From b538af41a5be5ab78f1d691de12d6975adb66d30 Mon Sep 17 00:00:00 2001 From: Stephen Hodgson Date: Mon, 29 Sep 2025 09:12:10 -0400 Subject: [PATCH 01/22] com.openai.unity 8.8.2 - bump deps --- OpenAI/Packages/com.openai.unity/package.json | 7 ++++--- OpenAI/Packages/manifest.json | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/OpenAI/Packages/com.openai.unity/package.json b/OpenAI/Packages/com.openai.unity/package.json index de0ba5ef..9c8bc405 100644 --- a/OpenAI/Packages/com.openai.unity/package.json +++ b/OpenAI/Packages/com.openai.unity/package.json @@ -3,7 +3,7 @@ "displayName": "OpenAI", "description": "A OpenAI package for the Unity to use though their RESTful API.\n\nIndependently developed, this is not an official library and I am not affiliated with OpenAI.\n\nAn OpenAI API account is required.", "keywords": [], - "version": "8.8.1", + "version": "8.8.2", "unity": "2021.3", "documentationUrl": "https://github.com/RageAgainstThePixel/com.openai.unity#documentation", "changelogUrl": "https://github.com/RageAgainstThePixel/com.openai.unity/releases", @@ -17,9 +17,10 @@ "url": "https://github.com/StephenHodgson" }, "dependencies": { + "com.utilities.audio": "2.2.8", "com.utilities.encoder.wav": "2.2.4", - "com.utilities.rest": "3.3.8", - "com.utilities.websockets": "1.0.5" + "com.utilities.rest": "4.0.0", + "com.utilities.websockets": "1.0.6" }, "samples": [ { diff --git a/OpenAI/Packages/manifest.json b/OpenAI/Packages/manifest.json index c13ef11a..e2998b18 100644 --- a/OpenAI/Packages/manifest.json +++ b/OpenAI/Packages/manifest.json @@ -1,10 +1,10 @@ { "dependencies": { - "com.unity.ide.rider": "3.0.36", + "com.unity.ide.rider": "3.0.38", "com.unity.ide.visualstudio": "2.0.23", "com.unity.textmeshpro": "3.0.9", "com.unity.ugui": "1.0.0", - "com.utilities.buildpipeline": "1.7.6" + "com.utilities.buildpipeline": "1.8.0" }, "scopedRegistries": [ { From 480dc78729bb905dd0fd13758a51b075117725a0 Mon Sep 17 00:00:00 2001 From: Lukasz D <12583808+dudziakl@users.noreply.github.com> Date: Mon, 29 Sep 2025 15:13:09 +0200 Subject: [PATCH 02/22] Fix a bug in parsing the Response on Android (#403) On Android Newtonsoft.Json cannot parse `IReadOnlyDictionary` field and fails to return a Response. Co-authored-by: Stephen Hodgson --- OpenAI/Packages/com.openai.unity/Runtime/Responses/Response.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Responses/Response.cs b/OpenAI/Packages/com.openai.unity/Runtime/Responses/Response.cs index 2ffa2a1f..4dcb0f3a 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Responses/Response.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Responses/Response.cs @@ -33,7 +33,7 @@ internal Response( [JsonProperty("parallel_tool_calls")] bool? parallelToolCalls = null, [JsonProperty("instructions")][JsonConverter(typeof(StringOrObjectConverter>))] object instructions = null, [JsonProperty("max_output_tokens")] int? maxOutputTokens = null, - [JsonProperty("metadata")] IReadOnlyDictionary metadata = null, + [JsonProperty("metadata")] Dictionary metadata = null, [JsonProperty("model")] string model = null, [JsonProperty("previous_response_id")] string previousResponseId = null, [JsonProperty("prompt")] Prompt prompt = null, From 04cb92cf960f2deaeb461cbd2fc629459535f037 Mon Sep 17 00:00:00 2001 From: Emily Eap Park Date: Mon, 29 Sep 2025 09:14:03 -0400 Subject: [PATCH 03/22] Added missing feature: Noise reduction settings for Realtime (#407) Added a new class NoiseReductionSettings and added it as a parameter to SessionConfiguration. Co-authored-by: Stephen Hodgson --- .../Realtime/NoiseReductionSettings.cs | 29 +++++++++++++++++++ .../Realtime/NoiseReductionSettings.cs.meta | 11 +++++++ .../Runtime/Realtime/Options.cs | 3 +- .../Runtime/Realtime/SessionConfiguration.cs | 23 +++++++++++---- 4 files changed, 60 insertions(+), 6 deletions(-) create mode 100644 OpenAI/Packages/com.openai.unity/Runtime/Realtime/NoiseReductionSettings.cs create mode 100644 OpenAI/Packages/com.openai.unity/Runtime/Realtime/NoiseReductionSettings.cs.meta diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Realtime/NoiseReductionSettings.cs b/OpenAI/Packages/com.openai.unity/Runtime/Realtime/NoiseReductionSettings.cs new file mode 100644 index 00000000..65ca574f --- /dev/null +++ b/OpenAI/Packages/com.openai.unity/Runtime/Realtime/NoiseReductionSettings.cs @@ -0,0 +1,29 @@ +using Newtonsoft.Json; +using System.Runtime.Serialization; +using UnityEngine.Scripting; + +namespace OpenAI.Realtime +{ + [Preserve] + public sealed class NoiseReductionSettings + { + public enum NoiseReduction + { + [EnumMember(Value = "near_field")] + near_field, + [EnumMember(Value = "far_field")] + far_field, + } + + [Preserve] + [JsonConstructor] + public NoiseReductionSettings(NoiseReduction type = NoiseReduction.near_field) + { + Type = type; + } + + [Preserve] + [JsonProperty("type", DefaultValueHandling = DefaultValueHandling.Ignore)] + public NoiseReduction Type { get; private set; } + } +} diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Realtime/NoiseReductionSettings.cs.meta b/OpenAI/Packages/com.openai.unity/Runtime/Realtime/NoiseReductionSettings.cs.meta new file mode 100644 index 00000000..2660956b --- /dev/null +++ b/OpenAI/Packages/com.openai.unity/Runtime/Realtime/NoiseReductionSettings.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: c106144d3e36d7a4a8ce1eb20603e10b +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Realtime/Options.cs b/OpenAI/Packages/com.openai.unity/Runtime/Realtime/Options.cs index 54672e87..0671a476 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Realtime/Options.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Realtime/Options.cs @@ -27,7 +27,8 @@ public static implicit operator SessionConfiguration(Options options) options.Tools, options.ToolChoice, options.Temperature, - options.MaxResponseOutputTokens); + options.MaxResponseOutputTokens, + null); public static implicit operator RealtimeResponseCreateParams(Options options) => new( diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Realtime/SessionConfiguration.cs b/OpenAI/Packages/com.openai.unity/Runtime/Realtime/SessionConfiguration.cs index 3bddaf1e..f60d4953 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Realtime/SessionConfiguration.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Realtime/SessionConfiguration.cs @@ -27,7 +27,8 @@ public SessionConfiguration( string toolChoice, float? temperature, int? maxResponseOutputTokens, - int? expiresAfter) + int? expiresAfter, + NoiseReductionSettings noiseReductionSettings) : this( model: model, modalities: modalities, @@ -41,7 +42,8 @@ public SessionConfiguration( toolChoice: toolChoice, temperature: temperature, maxResponseOutputTokens: maxResponseOutputTokens, - expiresAfter: expiresAfter) + expiresAfter: expiresAfter, + noiseReductionSettings: noiseReductionSettings) { } @@ -59,7 +61,8 @@ public SessionConfiguration( string toolChoice = null, float? temperature = null, int? maxResponseOutputTokens = null, - int? expiresAfter = null) + int? expiresAfter = null, + NoiseReductionSettings noiseReductionSettings = null) { ClientSecret = new ClientSecret(expiresAfter); Model = string.IsNullOrWhiteSpace(model?.Id) ? Models.Model.GPT4oRealtime : model; @@ -95,6 +98,8 @@ public SessionConfiguration( _ => maxResponseOutputTokens }; } + + InputAudioNoiseReduction = noiseReductionSettings; } [Preserve] @@ -110,7 +115,8 @@ internal SessionConfiguration( IReadOnlyList tools, object toolChoice, float? temperature, - object maxResponseOutputTokens) + object maxResponseOutputTokens, + NoiseReductionSettings noiseReductionSettings) { Model = model; Modalities = modalities; @@ -124,6 +130,7 @@ internal SessionConfiguration( ToolChoice = toolChoice; Temperature = temperature; MaxResponseOutputTokens = maxResponseOutputTokens; + InputAudioNoiseReduction = noiseReductionSettings; } [Preserve] @@ -141,7 +148,8 @@ internal SessionConfiguration( [JsonProperty("tools")] IReadOnlyList tools, [JsonProperty("tool_choice")] object toolChoice, [JsonProperty("temperature")] float? temperature, - [JsonProperty("max_response_output_tokens")] object maxResponseOutputTokens) + [JsonProperty("max_response_output_tokens")] object maxResponseOutputTokens, + [JsonProperty("input_audio_noise_reduction")] NoiseReductionSettings inputAudioNoiseReductionSettings) { ClientSecret = clientSecret; Modalities = modalities; @@ -156,6 +164,7 @@ internal SessionConfiguration( ToolChoice = toolChoice; Temperature = temperature; MaxResponseOutputTokens = maxResponseOutputTokens; + InputAudioNoiseReduction = inputAudioNoiseReductionSettings; } [Preserve] @@ -211,5 +220,9 @@ internal SessionConfiguration( [Preserve] [JsonProperty("max_response_output_tokens", DefaultValueHandling = DefaultValueHandling.Ignore)] public object MaxResponseOutputTokens { get; private set; } + + [Preserve] + [JsonProperty("input_audio_noise_reduction", DefaultValueHandling = DefaultValueHandling.Ignore)] + public NoiseReductionSettings InputAudioNoiseReduction { get; private set; } } } From a209111e2b21e3ca65926fdce592cb6ac1c74ac5 Mon Sep 17 00:00:00 2001 From: Emily Eap Park Date: Mon, 29 Sep 2025 09:14:41 -0400 Subject: [PATCH 04/22] Update ConversationItemTruncateRequest.cs (#408) - Changed how ContentIndex is serialized; ContentIndex should always be 0 which is the same as the default value Co-authored-by: Stephen Hodgson --- .../Runtime/Realtime/ConversationItemTruncateRequest.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Realtime/ConversationItemTruncateRequest.cs b/OpenAI/Packages/com.openai.unity/Runtime/Realtime/ConversationItemTruncateRequest.cs index 98ecc005..3fc4b6a8 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Realtime/ConversationItemTruncateRequest.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Realtime/ConversationItemTruncateRequest.cs @@ -47,7 +47,7 @@ public ConversationItemTruncateRequest(string itemId, int contentIndex, int audi /// The index of the content part to truncate. Set this to 0. /// [Preserve] - [JsonProperty("content_index")] + [JsonProperty("content_index", DefaultValueHandling = DefaultValueHandling.Include)] public int ContentIndex { get; } /// From 7c0ca530863a351ae524d413c10eb5bcfc9f3287 Mon Sep 17 00:00:00 2001 From: Stephen Hodgson Date: Mon, 29 Sep 2025 11:09:35 -0400 Subject: [PATCH 05/22] Add file_url for responses api --- .../Runtime/Responses/FileContent.cs | 23 +++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Responses/FileContent.cs b/OpenAI/Packages/com.openai.unity/Runtime/Responses/FileContent.cs index ce453456..2900f8c5 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Responses/FileContent.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Responses/FileContent.cs @@ -14,12 +14,14 @@ internal FileContent( [JsonProperty("type")] ResponseContentType type, [JsonProperty("file_data")] string fileData, [JsonProperty("file_id")] string fileId, - [JsonProperty("file_name")] string fileName) + [JsonProperty("file_name")] string fileName, + [JsonProperty("file_url")] string fileUrl) { Type = type; FileData = fileData; FileId = fileId; FileName = fileName; + FileUrl = fileUrl; } [Preserve] @@ -30,11 +32,24 @@ public FileContent(string fileName, string fileData) FileName = fileName; } + /// + /// The input file, can be a file id or a file url. + /// + /// The id or url of the file. + /// If the fileId starts with "http" or "https", it is a file url, otherwise it is a file id. [Preserve] public FileContent(string fileId) { Type = ResponseContentType.InputFile; - FileId = fileId; + + if (fileId.StartsWith("http")) + { + FileUrl = fileId; + } + else + { + FileId = fileId; + } } [Preserve] @@ -57,6 +72,10 @@ public FileContent(byte[] fileData, string fileName) [JsonProperty("file_id", DefaultValueHandling = DefaultValueHandling.Ignore)] public string FileId { get; } + [Preserve] + [JsonProperty("file_url", DefaultValueHandling = DefaultValueHandling.Ignore)] + public string FileUrl { get; private set; } + [Preserve] [JsonProperty("file_name", DefaultValueHandling = DefaultValueHandling.Ignore)] public string FileName { get; } From 1d2f13ddc1a50bedf3756ae83321f4031d3b2eb7 Mon Sep 17 00:00:00 2001 From: Stephen Hodgson Date: Mon, 29 Sep 2025 20:02:40 -0400 Subject: [PATCH 06/22] fix noise reduction impl --- .../Runtime/Realtime/NoiseReduction.cs | 16 ++++++++++++++++ .../Runtime/Realtime/NoiseReductionSettings.cs | 17 +++++------------ .../Runtime/Realtime/SessionConfiguration.cs | 6 ++---- 3 files changed, 23 insertions(+), 16 deletions(-) create mode 100644 OpenAI/Packages/com.openai.unity/Runtime/Realtime/NoiseReduction.cs diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Realtime/NoiseReduction.cs b/OpenAI/Packages/com.openai.unity/Runtime/Realtime/NoiseReduction.cs new file mode 100644 index 00000000..257f5c89 --- /dev/null +++ b/OpenAI/Packages/com.openai.unity/Runtime/Realtime/NoiseReduction.cs @@ -0,0 +1,16 @@ +// Licensed under the MIT License. See LICENSE in the project root for license information. + +using System.Runtime.Serialization; +using UnityEngine.Scripting; + +namespace OpenAI.Realtime +{ + [Preserve] + public enum NoiseReduction + { + [EnumMember(Value = "near_field")] + NearField, + [EnumMember(Value = "far_field")] + FarField, + } +} diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Realtime/NoiseReductionSettings.cs b/OpenAI/Packages/com.openai.unity/Runtime/Realtime/NoiseReductionSettings.cs index 65ca574f..e807ddce 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Realtime/NoiseReductionSettings.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Realtime/NoiseReductionSettings.cs @@ -1,5 +1,6 @@ -using Newtonsoft.Json; -using System.Runtime.Serialization; +// Licensed under the MIT License. See LICENSE in the project root for license information. + +using Newtonsoft.Json; using UnityEngine.Scripting; namespace OpenAI.Realtime @@ -7,23 +8,15 @@ namespace OpenAI.Realtime [Preserve] public sealed class NoiseReductionSettings { - public enum NoiseReduction - { - [EnumMember(Value = "near_field")] - near_field, - [EnumMember(Value = "far_field")] - far_field, - } - [Preserve] [JsonConstructor] - public NoiseReductionSettings(NoiseReduction type = NoiseReduction.near_field) + public NoiseReductionSettings([JsonProperty("type")] NoiseReduction type = NoiseReduction.NearField) { Type = type; } [Preserve] - [JsonProperty("type", DefaultValueHandling = DefaultValueHandling.Ignore)] + [JsonProperty("type", DefaultValueHandling = DefaultValueHandling.Include)] public NoiseReduction Type { get; private set; } } } diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Realtime/SessionConfiguration.cs b/OpenAI/Packages/com.openai.unity/Runtime/Realtime/SessionConfiguration.cs index f60d4953..06a37740 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Realtime/SessionConfiguration.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Realtime/SessionConfiguration.cs @@ -27,8 +27,7 @@ public SessionConfiguration( string toolChoice, float? temperature, int? maxResponseOutputTokens, - int? expiresAfter, - NoiseReductionSettings noiseReductionSettings) + int? expiresAfter) : this( model: model, modalities: modalities, @@ -42,8 +41,7 @@ public SessionConfiguration( toolChoice: toolChoice, temperature: temperature, maxResponseOutputTokens: maxResponseOutputTokens, - expiresAfter: expiresAfter, - noiseReductionSettings: noiseReductionSettings) + expiresAfter: expiresAfter) { } From d5a318d9725058923c3f4cb6bdbe000e24e8d430 Mon Sep 17 00:00:00 2001 From: Stephen Hodgson Date: Mon, 29 Sep 2025 22:28:28 -0400 Subject: [PATCH 07/22] added response conversation endpoint refactored json object ctrs to fix AOT serialization problems added support for gpt-5 and latest API changes --- .../Runtime/Assistants/AssistantResponse.cs | 2 +- .../Runtime/Batch/BatchErrors.cs | 2 +- .../Runtime/Batch/BatchResponse.cs | 2 +- .../Runtime/Chat/ChatResponse.cs | 2 +- .../com.openai.unity/Runtime/Chat/Delta.cs | 2 +- .../Runtime/Common/ReasoningEffort.cs | 6 +- .../Runtime/ConversationsEndpoint.cs | 219 ++++++++++++++++++ .../Runtime/ConversationsEndpoint.cs.meta | 11 + .../Runtime/CreateConversationRequest.cs | 47 ++++ .../Runtime/CreateConversationRequest.cs.meta | 11 + .../Runtime/Embeddings/Datum.cs | 2 +- .../Runtime/Embeddings/EmbeddingsResponse.cs | 2 +- .../Runtime/Images/ImagesResponse.cs | 2 +- .../com.openai.unity/Runtime/Models/Model.cs | 2 +- .../Moderations/ModerationsResponse.cs | 2 +- .../com.openai.unity/Runtime/OpenAIClient.cs | 7 + .../Runtime/Realtime/ConversationItem.cs | 2 +- .../Runtime/Realtime/NoiseReduction.cs.meta | 11 + .../Realtime/NoiseReductionSettings.cs.meta | 2 +- .../Runtime/Realtime/Options.cs | 2 +- .../Runtime/Realtime/RateLimitsResponse.cs | 2 +- .../Realtime/RealtimeResponseResource.cs | 4 +- .../Runtime/Realtime/SessionConfiguration.cs | 2 +- .../Responses/CodeInterpreterToolCall.cs | 2 +- .../Runtime/Responses/ComputerToolCall.cs | 2 +- .../Runtime/Responses/Conversation.cs | 63 +++++ .../Runtime/Responses/Conversation.cs.meta | 11 + .../Responses/CreateResponseRequest.cs | 68 +++++- .../Runtime/Responses/DragComputerAction.cs | 2 +- .../Runtime/Responses/FileSearchResult.cs | 2 +- .../Runtime/Responses/FileSearchTool.cs | 4 +- .../Runtime/Responses/FileSearchToolCall.cs | 4 +- .../Responses/KeyPressComputerAction.cs | 2 +- .../Runtime/Responses/LocalShellAction.cs | 4 +- .../Runtime/Responses/MCPListTools.cs | 2 +- .../Runtime/Responses/MCPTool.cs | 4 +- .../Runtime/Responses/Message.cs | 2 +- .../Runtime/Responses/Prompt.cs | 2 +- .../Runtime/Responses/ReasoningContent.cs | 53 +++++ .../Responses/ReasoningContent.cs.meta | 11 + .../Runtime/Responses/ReasoningItem.cs | 46 +++- .../Runtime/Responses/ReasoningSummary.cs | 2 +- .../Runtime/Responses/Response.cs | 58 ++++- .../Runtime/Responses/ResponseContentType.cs | 2 +- .../Runtime/Responses/ResponsesEndpoint.cs | 44 +++- .../Runtime/Responses/TextContent.cs | 4 +- .../Runtime/Threads/CodeInterpreterOutputs.cs | 2 +- .../Runtime/Threads/Message.cs | 4 +- .../Runtime/Threads/MessageDelta.cs | 2 +- .../Runtime/Threads/MessageResponse.cs | 4 +- .../Runtime/Threads/SubmitToolOutputs.cs | 2 +- .../Runtime/Threads/ToolCall.cs | 2 +- 52 files changed, 696 insertions(+), 60 deletions(-) create mode 100644 OpenAI/Packages/com.openai.unity/Runtime/ConversationsEndpoint.cs create mode 100644 OpenAI/Packages/com.openai.unity/Runtime/ConversationsEndpoint.cs.meta create mode 100644 OpenAI/Packages/com.openai.unity/Runtime/CreateConversationRequest.cs create mode 100644 OpenAI/Packages/com.openai.unity/Runtime/CreateConversationRequest.cs.meta create mode 100644 OpenAI/Packages/com.openai.unity/Runtime/Realtime/NoiseReduction.cs.meta create mode 100644 OpenAI/Packages/com.openai.unity/Runtime/Responses/Conversation.cs create mode 100644 OpenAI/Packages/com.openai.unity/Runtime/Responses/Conversation.cs.meta create mode 100644 OpenAI/Packages/com.openai.unity/Runtime/Responses/ReasoningContent.cs create mode 100644 OpenAI/Packages/com.openai.unity/Runtime/Responses/ReasoningContent.cs.meta diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Assistants/AssistantResponse.cs b/OpenAI/Packages/com.openai.unity/Runtime/Assistants/AssistantResponse.cs index 9cc651fb..773ef2a5 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Assistants/AssistantResponse.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Assistants/AssistantResponse.cs @@ -24,7 +24,7 @@ internal AssistantResponse( [JsonProperty("description")] string description, [JsonProperty("model")] string model, [JsonProperty("instructions")] string instructions, - [JsonProperty("tools")] IReadOnlyList tools, + [JsonProperty("tools")] List tools, [JsonProperty("tool_resources")] ToolResources toolResources, [JsonProperty("metadata")] Dictionary metadata, [JsonProperty("temperature")] float? temperature, diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Batch/BatchErrors.cs b/OpenAI/Packages/com.openai.unity/Runtime/Batch/BatchErrors.cs index 500d050e..264a0b48 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Batch/BatchErrors.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Batch/BatchErrors.cs @@ -11,7 +11,7 @@ public sealed class BatchErrors { [Preserve] [JsonConstructor] - internal BatchErrors([JsonProperty("data")] IReadOnlyList errors) + internal BatchErrors([JsonProperty("data")] List errors) { Errors = errors; } diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Batch/BatchResponse.cs b/OpenAI/Packages/com.openai.unity/Runtime/Batch/BatchResponse.cs index 35323d11..3c72d610 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Batch/BatchResponse.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Batch/BatchResponse.cs @@ -31,7 +31,7 @@ internal BatchResponse( [JsonProperty("expired_at")] int? expiredAt, [JsonProperty("cancelled_at")] int? cancelledAt, [JsonProperty("request_counts")] RequestCounts requestCounts, - [JsonProperty("metadata")] IReadOnlyDictionary metadata) + [JsonProperty("metadata")] Dictionary metadata) { Id = id; Object = @object; diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Chat/ChatResponse.cs b/OpenAI/Packages/com.openai.unity/Runtime/Chat/ChatResponse.cs index 08414afa..cc7833fe 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Chat/ChatResponse.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Chat/ChatResponse.cs @@ -27,7 +27,7 @@ internal ChatResponse( [JsonProperty("service_tier")] string serviceTier, [JsonProperty("system_fingerprint")] string systemFingerprint, [JsonProperty("usage")] Usage usage, - [JsonProperty("choices")] IReadOnlyList choices) + [JsonProperty("choices")] List choices) { Id = id; Object = @object; diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Chat/Delta.cs b/OpenAI/Packages/com.openai.unity/Runtime/Chat/Delta.cs index e9b9eaad..b8017c98 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Chat/Delta.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Chat/Delta.cs @@ -18,7 +18,7 @@ public Delta( [JsonProperty("content")] string content, [JsonProperty("refusal")] string refusal, [JsonProperty("name")] string name, - [JsonProperty("function_call")] IReadOnlyList toolCalls) + [JsonProperty("function_call")] List toolCalls) { Role = role; Content = content; diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Common/ReasoningEffort.cs b/OpenAI/Packages/com.openai.unity/Runtime/Common/ReasoningEffort.cs index ab2da4e3..db0193fc 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Common/ReasoningEffort.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Common/ReasoningEffort.cs @@ -6,15 +6,17 @@ namespace OpenAI { /// /// Constrains the effort of reasoning for Reasoning Models.
- /// Currently supported values are: Low, Medium, High. Reducing reasoning effort can result in faster responses and fewer tokens used on reasoning response. + /// Currently supported values are: Minimal, Low, Medium, High. Reducing reasoning effort can result in faster responses and fewer tokens used on reasoning response. /// /// /// Reasoning models only! /// public enum ReasoningEffort { + [EnumMember(Value = "minimal")] + Minimal = 1, [EnumMember(Value = "low")] - Low = 1, + Low, [EnumMember(Value = "medium")] Medium, [EnumMember(Value = "high")] diff --git a/OpenAI/Packages/com.openai.unity/Runtime/ConversationsEndpoint.cs b/OpenAI/Packages/com.openai.unity/Runtime/ConversationsEndpoint.cs new file mode 100644 index 00000000..6de3bfed --- /dev/null +++ b/OpenAI/Packages/com.openai.unity/Runtime/ConversationsEndpoint.cs @@ -0,0 +1,219 @@ +// Licensed under the MIT License. See LICENSE in the project root for license information. + +using Newtonsoft.Json; +using OpenAI.Extensions; +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using UnityEngine.TextCore.Text; +using Utilities.WebRequestRest; + +namespace OpenAI.Responses +{ + public sealed class ConversationsEndpoint : OpenAIBaseEndpoint + { + public ConversationsEndpoint(OpenAIClient client) : base(client) { } + + protected override string Root => "conversations"; + + /// + /// Create a conversation. + /// + /// . + /// Optional, . + /// . + public async Task CreateConversationAsync(CreateConversationRequest request, CancellationToken cancellationToken = default) + { + var payload = JsonConvert.SerializeObject(request, OpenAIClient.JsonSerializationOptions); + var response = await Rest.PostAsync(GetUrl(), payload, new RestParameters(client.DefaultRequestHeaders), cancellationToken); + response.Validate(EnableDebug); + return response.Deserialize(client); + } + + /// + /// Get a conversation. + /// + /// The id of the conversation to retrieve. + /// Optional, . + /// . + public async Task GetConversationAsync(string conversationId, CancellationToken cancellationToken = default) + { + if (string.IsNullOrWhiteSpace(conversationId)) + { + throw new ArgumentNullException(nameof(conversationId)); + } + + var response = await Rest.GetAsync(GetUrl($"/{conversationId}"), new RestParameters(client.DefaultRequestHeaders), cancellationToken); + response.Validate(EnableDebug); + return response.Deserialize(client); + } + + /// + /// Update a conversation. + /// + /// + /// The id of the conversation to retrieve. + /// + /// + /// Set of 16 key-value pairs that can be attached to an object. + /// This can be useful for storing additional information about the object in a structured format, + /// and querying for objects via API or the dashboard. + /// Keys are strings with a maximum length of 64 characters.Values are strings with a maximum length of 512 characters. + /// + /// Optional, . + /// . + public async Task UpdateConversationAsync(string conversationId, IReadOnlyDictionary metadata, CancellationToken cancellationToken = default) + { + if (string.IsNullOrWhiteSpace(conversationId)) + { + throw new ArgumentNullException(nameof(conversationId)); + } + + var payload = JsonConvert.SerializeObject(new { metadata }, OpenAIClient.JsonSerializationOptions); + var response = await Rest.PatchAsync(GetUrl($"/{conversationId}"), payload, new RestParameters(client.DefaultRequestHeaders), cancellationToken); + response.Validate(EnableDebug); + return response.Deserialize(client); + } + + /// + /// Delete a conversation. + /// + /// + /// Items in the conversation will not be deleted. + /// + /// The id of the conversation to retrieve. + /// Optional, . + /// True, if the was deleted successfully, otherwise False. + public async Task DeleteConversationAsync(string conversationId, CancellationToken cancellationToken = default) + { + if (string.IsNullOrWhiteSpace(conversationId)) + { + throw new ArgumentNullException(nameof(conversationId)); + } + + var response = await Rest.DeleteAsync(GetUrl($"/{conversationId}"), new RestParameters(client.DefaultRequestHeaders), cancellationToken); + response.Validate(EnableDebug); + var result = response.Deserialize(client); + return result.Deleted; + } + + #region Conversation Items + + /// + /// List all items for a conversation with the given ID. + /// + /// The ID of the conversation to list items for. + /// Optional, . + /// Optional, . + /// . + public async Task> ListConversationItemsAsync(string conversationId, ListQuery query = null, CancellationToken cancellationToken = default) + { + if (string.IsNullOrWhiteSpace(conversationId)) + { + throw new ArgumentNullException(nameof(conversationId)); + } + + var response = await Rest.GetAsync(GetUrl($"/{conversationId}/items", query), new RestParameters(client.DefaultRequestHeaders), cancellationToken); + response.Validate(EnableDebug); + return response.Deserialize>(client); + } + + /// + /// Create items in a conversation with the given ID. + /// + /// The ID of the conversation to add the item to. + /// The items to add to the conversation. You may add up to 20 items at a time. + /// Optional, Additional fields to include in the response. + /// Optional, . + /// . + public async Task> CreateConversationItemsAsync(string conversationId, IEnumerable items, string[] include = null, CancellationToken cancellationToken = default) + { + if (string.IsNullOrWhiteSpace(conversationId)) + { + throw new ArgumentNullException(nameof(conversationId)); + } + + if (items == null) + { + throw new ArgumentNullException(nameof(items)); + } + + var payload = JsonConvert.SerializeObject(new { items }, OpenAIClient.JsonSerializationOptions); + Dictionary query = null; + + if (include != null) + { + query = new Dictionary + { + { "include", string.Join(",", include) } + }; + } + + var response = await Rest.PostAsync(GetUrl($"/{conversationId}/items", query), payload, new RestParameters(client.DefaultRequestHeaders), cancellationToken); + response.Validate(EnableDebug); + return response.Deserialize>(client); + } + + /// + /// Retrieve an item from a conversation. + /// + /// The ID of the conversation that contains the item. + /// The ID of the item to retrieve. + /// Optional, Additional fields to include in the response. + /// Optional, . + /// . + public async Task GetConversationItemAsync(string conversationId, string itemId, string[] include = null, CancellationToken cancellationToken = default) + { + if (string.IsNullOrWhiteSpace(conversationId)) + { + throw new ArgumentNullException(nameof(conversationId)); + } + + if (string.IsNullOrWhiteSpace(itemId)) + { + throw new ArgumentNullException(nameof(itemId)); + } + + Dictionary query = null; + + if (include != null) + { + query = new Dictionary + { + { "include", string.Join(",", include) } + }; + } + + var response = await Rest.GetAsync(GetUrl($"/{conversationId}/items/{itemId}", query), new RestParameters(client.DefaultRequestHeaders), cancellationToken); + response.Validate(EnableDebug); + return response.Deserialize(client); + } + + /// + /// Delete an item from a conversation with the given IDs. + /// + /// The ID of the conversation that contains the item. + /// The ID of the item to delete. + /// Optional, . + /// Returns the updated >. + public async Task DeleteConversationItemAsync(string conversationId, string itemId, CancellationToken cancellationToken = default) + { + if (string.IsNullOrWhiteSpace(conversationId)) + { + throw new ArgumentNullException(nameof(conversationId)); + } + + if (string.IsNullOrWhiteSpace(itemId)) + { + throw new ArgumentNullException(nameof(itemId)); + } + + var response = await Rest.DeleteAsync(GetUrl($"/{conversationId}/items/{itemId}"), new RestParameters(client.DefaultRequestHeaders), cancellationToken); + response.Validate(EnableDebug); + return response.Deserialize(client); + } + + #endregion Conversation Items + } +} diff --git a/OpenAI/Packages/com.openai.unity/Runtime/ConversationsEndpoint.cs.meta b/OpenAI/Packages/com.openai.unity/Runtime/ConversationsEndpoint.cs.meta new file mode 100644 index 00000000..7502345c --- /dev/null +++ b/OpenAI/Packages/com.openai.unity/Runtime/ConversationsEndpoint.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 53f75d424ac936f4483d8e8f381748b2 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {fileID: 2800000, guid: 84a7eb8fc6eba7540bf56cea8e12249c, type: 3} + userData: + assetBundleName: + assetBundleVariant: diff --git a/OpenAI/Packages/com.openai.unity/Runtime/CreateConversationRequest.cs b/OpenAI/Packages/com.openai.unity/Runtime/CreateConversationRequest.cs new file mode 100644 index 00000000..bd6fef2b --- /dev/null +++ b/OpenAI/Packages/com.openai.unity/Runtime/CreateConversationRequest.cs @@ -0,0 +1,47 @@ +// Licensed under the MIT License. See LICENSE in the project root for license information. + +using Newtonsoft.Json; +using System.Collections.Generic; +using UnityEngine.Scripting; + +namespace OpenAI.Responses +{ + [Preserve] + public sealed class CreateConversationRequest + { + public CreateConversationRequest(IResponseItem item, IReadOnlyDictionary metadata = null) + : this(new[] { item }, metadata) + { + } + + public CreateConversationRequest(IEnumerable items = null, IReadOnlyDictionary metadata = null) + { + Items = items; + Metadata = metadata; + } + + [JsonConstructor] + internal CreateConversationRequest(List items, Dictionary metadata) + { + Items = items; + Metadata = metadata; + } + + /// + /// Initial items to include in the conversation context. You may add up to 20 items at a time. + /// + [Preserve] + [JsonProperty("items", DefaultValueHandling = DefaultValueHandling.Ignore)] + public IEnumerable Items { get; set; } + + /// + /// Set of 16 key-value pairs that can be attached to an object. + /// This can be useful for storing additional information about the object in a structured format, + /// and querying for objects via API or the dashboard. Keys are strings with a maximum length of 64 characters. + /// Values are strings with a maximum length of 512 characters. + /// + [Preserve] + [JsonProperty("metadata", DefaultValueHandling = DefaultValueHandling.Ignore)] + public IReadOnlyDictionary Metadata { get; set; } + } +} diff --git a/OpenAI/Packages/com.openai.unity/Runtime/CreateConversationRequest.cs.meta b/OpenAI/Packages/com.openai.unity/Runtime/CreateConversationRequest.cs.meta new file mode 100644 index 00000000..8d4199bd --- /dev/null +++ b/OpenAI/Packages/com.openai.unity/Runtime/CreateConversationRequest.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: c758cbaa906012b4daef794aaba6c241 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {fileID: 2800000, guid: 84a7eb8fc6eba7540bf56cea8e12249c, type: 3} + userData: + assetBundleName: + assetBundleVariant: diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Embeddings/Datum.cs b/OpenAI/Packages/com.openai.unity/Runtime/Embeddings/Datum.cs index 090bc132..af620d5f 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Embeddings/Datum.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Embeddings/Datum.cs @@ -13,7 +13,7 @@ public sealed class Datum [JsonConstructor] internal Datum( [JsonProperty("object")] string @object, - [JsonProperty("embedding")] IReadOnlyList embedding, + [JsonProperty("embedding")] List embedding, [JsonProperty("index")] int index) { Object = @object; diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Embeddings/EmbeddingsResponse.cs b/OpenAI/Packages/com.openai.unity/Runtime/Embeddings/EmbeddingsResponse.cs index 0a3c4898..6ac1d8e8 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Embeddings/EmbeddingsResponse.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Embeddings/EmbeddingsResponse.cs @@ -13,7 +13,7 @@ public sealed class EmbeddingsResponse : BaseResponse [JsonConstructor] internal EmbeddingsResponse( [JsonProperty("object")] string @object, - [JsonProperty("data")] IReadOnlyList data, + [JsonProperty("data")] List data, [JsonProperty("model")] string model, [JsonProperty("usage")] Usage usage) { diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Images/ImagesResponse.cs b/OpenAI/Packages/com.openai.unity/Runtime/Images/ImagesResponse.cs index 7dc854b8..5e26dfa3 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Images/ImagesResponse.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Images/ImagesResponse.cs @@ -13,7 +13,7 @@ internal sealed class ImagesResponse : BaseResponse [JsonConstructor] internal ImagesResponse( [JsonProperty("created")] long createdAtUnixSeconds, - [JsonProperty("data")] IReadOnlyList results, + [JsonProperty("data")] List results, [JsonProperty("background")] string background, [JsonProperty("output_format")] string outputFormat, [JsonProperty("quality")] string quality, diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Models/Model.cs b/OpenAI/Packages/com.openai.unity/Runtime/Models/Model.cs index 6cfdaff8..01239399 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Models/Model.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Models/Model.cs @@ -38,7 +38,7 @@ internal Model( [JsonProperty("object")] string @object, [JsonProperty("created")] int createdAtUnixTimeSeconds, [JsonProperty("owned_by")] string ownedBy, - [JsonProperty("permission")] IReadOnlyList permissions, + [JsonProperty("permission")] List permissions, [JsonProperty("root")] string root, [JsonProperty("parent")] string parent) : this(id) diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Moderations/ModerationsResponse.cs b/OpenAI/Packages/com.openai.unity/Runtime/Moderations/ModerationsResponse.cs index f6164191..006dc978 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Moderations/ModerationsResponse.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Moderations/ModerationsResponse.cs @@ -14,7 +14,7 @@ public sealed class ModerationsResponse : BaseResponse internal ModerationsResponse( [JsonProperty("id")] string id, [JsonProperty("model")] string model, - [JsonProperty("results")] IReadOnlyList results) + [JsonProperty("results")] List results) { Id = id; Model = model; diff --git a/OpenAI/Packages/com.openai.unity/Runtime/OpenAIClient.cs b/OpenAI/Packages/com.openai.unity/Runtime/OpenAIClient.cs index 27647d92..542c23ab 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/OpenAIClient.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/OpenAIClient.cs @@ -63,6 +63,7 @@ public OpenAIClient(OpenAIAuthentication authentication = null, OpenAISettings s VectorStoresEndpoint = new VectorStoresEndpoint(this); RealtimeEndpoint = new RealtimeEndpoint(this); ResponsesEndpoint = new ResponsesEndpoint(this); + ConversationsEndpoint = new ConversationsEndpoint(this); } protected override void SetupDefaultRequestHeaders() @@ -235,6 +236,12 @@ protected override void ValidateAuthentication() /// public ResponsesEndpoint ResponsesEndpoint { get; } + /// + /// Create and manage conversations to store and retrieve conversation state across Response API calls. + /// + /// + public ConversationsEndpoint ConversationsEndpoint { get; } + #endregion Endpoints } } diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Realtime/ConversationItem.cs b/OpenAI/Packages/com.openai.unity/Runtime/Realtime/ConversationItem.cs index 1b44c6ae..2c69819a 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Realtime/ConversationItem.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Realtime/ConversationItem.cs @@ -20,7 +20,7 @@ internal ConversationItem( [JsonProperty("object")] string @object, [JsonProperty("status")] RealtimeResponseStatus status, [JsonProperty("role")] Role role, - [JsonProperty("content")] IReadOnlyList content, + [JsonProperty("content")] List content, [JsonProperty("call_id")] string functionCallId, [JsonProperty("name")] string functionName, [JsonProperty("arguments")] string functionArguments, diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Realtime/NoiseReduction.cs.meta b/OpenAI/Packages/com.openai.unity/Runtime/Realtime/NoiseReduction.cs.meta new file mode 100644 index 00000000..49a0c39d --- /dev/null +++ b/OpenAI/Packages/com.openai.unity/Runtime/Realtime/NoiseReduction.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 4a1ce1cbabf61074da015bc49810a116 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {fileID: 2800000, guid: 84a7eb8fc6eba7540bf56cea8e12249c, type: 3} + userData: + assetBundleName: + assetBundleVariant: diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Realtime/NoiseReductionSettings.cs.meta b/OpenAI/Packages/com.openai.unity/Runtime/Realtime/NoiseReductionSettings.cs.meta index 2660956b..0857822b 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Realtime/NoiseReductionSettings.cs.meta +++ b/OpenAI/Packages/com.openai.unity/Runtime/Realtime/NoiseReductionSettings.cs.meta @@ -5,7 +5,7 @@ MonoImporter: serializedVersion: 2 defaultReferences: [] executionOrder: 0 - icon: {instanceID: 0} + icon: {fileID: 2800000, guid: 84a7eb8fc6eba7540bf56cea8e12249c, type: 3} userData: assetBundleName: assetBundleVariant: diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Realtime/Options.cs b/OpenAI/Packages/com.openai.unity/Runtime/Realtime/Options.cs index 0671a476..2b9ae9c3 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Realtime/Options.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Realtime/Options.cs @@ -54,7 +54,7 @@ internal Options( [JsonProperty("output_audio_format")] RealtimeAudioFormat outputAudioFormat, [JsonProperty("input_audio_transcription")] InputAudioTranscriptionSettings inputAudioTranscriptionSettings, [JsonProperty("turn_detection")] VoiceActivityDetectionSettings voiceActivityDetectionSettings, - [JsonProperty("tools")] IReadOnlyList tools, + [JsonProperty("tools")] List tools, [JsonProperty("tool_choice")] object toolChoice, [JsonProperty("temperature")] float? temperature, [JsonProperty("max_response_output_tokens")] object maxResponseOutputTokens) diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Realtime/RateLimitsResponse.cs b/OpenAI/Packages/com.openai.unity/Runtime/Realtime/RateLimitsResponse.cs index 7f268fa3..6ec24f77 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Realtime/RateLimitsResponse.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Realtime/RateLimitsResponse.cs @@ -14,7 +14,7 @@ public sealed class RateLimitsResponse : BaseRealtimeEvent, IServerEvent internal RateLimitsResponse( [JsonProperty("event_id")] string eventId, [JsonProperty("type")] string type, - [JsonProperty("rate_limits")] IReadOnlyList rateLimits) + [JsonProperty("rate_limits")] List rateLimits) { EventId = eventId; Type = type; diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Realtime/RealtimeResponseResource.cs b/OpenAI/Packages/com.openai.unity/Runtime/Realtime/RealtimeResponseResource.cs index e47c9866..4a07de3f 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Realtime/RealtimeResponseResource.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Realtime/RealtimeResponseResource.cs @@ -17,8 +17,8 @@ internal RealtimeResponseResource( [JsonProperty("object")] string @object, [JsonProperty("status")] RealtimeResponseStatus status, [JsonProperty("status_details")] StatusDetails statusDetails, - [JsonProperty("output")] IReadOnlyList output, - [JsonProperty("metadata")] IReadOnlyDictionary metadata, + [JsonProperty("output")] List output, + [JsonProperty("metadata")] Dictionary metadata, [JsonProperty("usage")] TokenUsage usage, [JsonProperty("conversation_id")] string conversationId, [JsonProperty("voice")] string voice, diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Realtime/SessionConfiguration.cs b/OpenAI/Packages/com.openai.unity/Runtime/Realtime/SessionConfiguration.cs index 06a37740..fcd08b4e 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Realtime/SessionConfiguration.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Realtime/SessionConfiguration.cs @@ -143,7 +143,7 @@ internal SessionConfiguration( [JsonProperty("output_audio_format")] RealtimeAudioFormat outputAudioFormat, [JsonProperty("input_audio_transcription")] InputAudioTranscriptionSettings inputAudioTranscriptionSettings, [JsonProperty("turn_detection")][JsonConverter(typeof(VoiceActivityDetectionSettingsConverter))] IVoiceActivityDetectionSettings voiceActivityDetectionSettings, - [JsonProperty("tools")] IReadOnlyList tools, + [JsonProperty("tools")] List tools, [JsonProperty("tool_choice")] object toolChoice, [JsonProperty("temperature")] float? temperature, [JsonProperty("max_response_output_tokens")] object maxResponseOutputTokens, diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Responses/CodeInterpreterToolCall.cs b/OpenAI/Packages/com.openai.unity/Runtime/Responses/CodeInterpreterToolCall.cs index f8eb3530..8670fc0d 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Responses/CodeInterpreterToolCall.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Responses/CodeInterpreterToolCall.cs @@ -21,7 +21,7 @@ internal CodeInterpreterToolCall( [JsonProperty("object")] string @object, [JsonProperty("status")] ResponseStatus status, [JsonProperty("code")] string code, - [JsonProperty("results")] IReadOnlyList results, + [JsonProperty("results")] List results, [JsonProperty("container_id")] string containerId) { Id = id; diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Responses/ComputerToolCall.cs b/OpenAI/Packages/com.openai.unity/Runtime/Responses/ComputerToolCall.cs index c777192b..a5844f20 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Responses/ComputerToolCall.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Responses/ComputerToolCall.cs @@ -21,7 +21,7 @@ internal ComputerToolCall( [JsonProperty("status")] ResponseStatus status, [JsonProperty("call_id")] string callId, [JsonProperty("action")] IComputerAction action, - [JsonProperty("pending_safety_checks")] IReadOnlyList pendingSafetyChecks, + [JsonProperty("pending_safety_checks")] List pendingSafetyChecks, [JsonProperty("output")] ComputerScreenShot computerScreenShot) { Id = id; diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Responses/Conversation.cs b/OpenAI/Packages/com.openai.unity/Runtime/Responses/Conversation.cs new file mode 100644 index 00000000..40af95c3 --- /dev/null +++ b/OpenAI/Packages/com.openai.unity/Runtime/Responses/Conversation.cs @@ -0,0 +1,63 @@ +// Licensed under the MIT License. See LICENSE in the project root for license information. + +using Newtonsoft.Json; +using System; +using System.Collections.Generic; +using UnityEngine.Scripting; + +namespace OpenAI.Responses +{ + [Preserve] + public sealed class Conversation + { + [JsonConstructor] + internal Conversation( + [JsonProperty("created_at")] long createdAtUnitTimeSeconds, + [JsonProperty("id")] string id, + [JsonProperty("metadata")] Dictionary metaData = null, + [JsonProperty("object")] string @object = null) + { + CreatedAtUnitTimeSeconds = createdAtUnitTimeSeconds; + Id = id; + MetaData = metaData; + Object = @object; + } + + /// + /// The time at which the conversation was created, measured in seconds since the Unix epoch. + /// + [Preserve] + [JsonProperty("created_at")] + public long CreatedAtUnitTimeSeconds { get; } + + /// + /// The datetime at which the conversation was created. + /// + [JsonIgnore] + public DateTime CreatedAt + => DateTimeOffset.FromUnixTimeSeconds(CreatedAtUnitTimeSeconds).UtcDateTime; + + /// + /// The unique ID of the conversation. + /// + [Preserve] + [JsonProperty("id")] + public string Id { get; } + + [Preserve] + [JsonProperty("metadata", DefaultValueHandling = DefaultValueHandling.Ignore)] + public IReadOnlyDictionary MetaData { get; } + + /// + /// The object type, which is always conversation. + /// + [Preserve] + [JsonProperty("object")] + public string Object { get; } + + public override string ToString() => Id; + + [Preserve] + public static implicit operator string(Conversation conversation) => conversation?.Id; + } +} diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Responses/Conversation.cs.meta b/OpenAI/Packages/com.openai.unity/Runtime/Responses/Conversation.cs.meta new file mode 100644 index 00000000..1740e277 --- /dev/null +++ b/OpenAI/Packages/com.openai.unity/Runtime/Responses/Conversation.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 506fa49151ce97743b4814aeb50d22de +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {fileID: 2800000, guid: 84a7eb8fc6eba7540bf56cea8e12249c, type: 3} + userData: + assetBundleName: + assetBundleVariant: diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Responses/CreateResponseRequest.cs b/OpenAI/Packages/com.openai.unity/Runtime/Responses/CreateResponseRequest.cs index 9d4754b7..a5e767f4 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Responses/CreateResponseRequest.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Responses/CreateResponseRequest.cs @@ -39,7 +39,12 @@ public CreateResponseRequest( IEnumerable tools = null, double? topP = null, Truncation truncation = Truncation.Auto, - string user = null) + string user = null, + string conversationId = null, + int? maxToolCalls = null, + string promptCacheKey = null, + string safetyIdentifier = null, + int? topLogProbs = null) : this( input: new List { new Message(Role.User, new TextContent(textInput)) }, model: model, @@ -61,7 +66,12 @@ public CreateResponseRequest( tools: tools, topP: topP, truncation: truncation, - user: user) + user: user, + conversationId: conversationId, + maxToolCalls: maxToolCalls, + promptCacheKey: promptCacheKey, + safetyIdentifier: safetyIdentifier, + topLogProbs: topLogProbs) { } @@ -87,7 +97,12 @@ public CreateResponseRequest( IEnumerable tools = null, double? topP = null, Truncation truncation = Truncation.Auto, - string user = null) + string user = null, + string conversationId = null, + int? maxToolCalls = null, + string promptCacheKey = null, + string safetyIdentifier = null, + int? topLogProbs = null) { Input = input?.ToArray() ?? throw new ArgumentNullException(nameof(input)); Model = string.IsNullOrWhiteSpace(model) ? Models.Model.ChatGPT4o : model; @@ -122,8 +137,13 @@ public CreateResponseRequest( Tools = toolList; ToolChoice = activeTool; TopP = topP; + TopLogProbs = topLogProbs; Truncation = truncation; User = user; + ConversationId = conversationId; + MaxToolCalls = maxToolCalls; + PromptCacheKey = promptCacheKey; + SafetyIdentifier = safetyIdentifier; } /// @@ -294,6 +314,14 @@ public CreateResponseRequest( [JsonProperty("top_p", DefaultValueHandling = DefaultValueHandling.Ignore)] public double? TopP { get; } + /// + /// An integer between 0 and 20 specifying the number of most likely tokens to return at each token position, + /// each with an associated log probability. + /// + [Preserve] + [JsonProperty("top_logprobs", DefaultValueHandling = DefaultValueHandling.Ignore)] + public int? TopLogProbs { get; } + /// /// The truncation strategy to use for the model response.
/// - Auto: If the context of this response and previous ones exceeds the model's context window size, @@ -310,5 +338,39 @@ public CreateResponseRequest( [Preserve] [JsonProperty("user", DefaultValueHandling = DefaultValueHandling.Ignore)] public string User { get; } + + /// + /// The conversation id that this response belongs to. + /// Items from this conversation are prepended to `input_items` for this response request. + /// Input items and output items from this response are automatically added to this conversation after this response completes. + /// + [Preserve] + [JsonProperty("conversation", DefaultValueHandling = DefaultValueHandling.Ignore)] + public string ConversationId { get; } + + /// + /// The maximum number of total calls to built-in tools that can be processed in a response. + /// This maximum number applies across all built-in tool calls, not per individual tool. + /// Any further attempts to call a tool by the model will be ignored. + /// + [Preserve] + [JsonProperty("max_tool_calls", DefaultValueHandling = DefaultValueHandling.Ignore)] + public int? MaxToolCalls { get; } + + /// + /// Used by OpenAI to cache responses for similar requests to optimize your cache hit rates. Replaces the user field. + /// + [Preserve] + [JsonProperty("prompt_cache_key", DefaultValueHandling = DefaultValueHandling.Ignore)] + public string PromptCacheKey { get; } + + /// + /// A stable identifier used to help detect users of your application that may be violating OpenAI's usage policies. + /// The IDs should be a string that uniquely identifies each user. + /// We recommend hashing their username or email address, in order to avoid sending us any identifying information. + /// + [Preserve] + [JsonProperty("safety_identifier", DefaultValueHandling = DefaultValueHandling.Ignore)] + public string SafetyIdentifier { get; } } } diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Responses/DragComputerAction.cs b/OpenAI/Packages/com.openai.unity/Runtime/Responses/DragComputerAction.cs index f2c4c55a..18dad17e 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Responses/DragComputerAction.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Responses/DragComputerAction.cs @@ -18,7 +18,7 @@ public sealed class DragComputerAction : IComputerAction [JsonConstructor] internal DragComputerAction( [JsonProperty("type")] ComputerActionType type, - [JsonProperty("path")] IReadOnlyList path) + [JsonProperty("path")] List path) { Type = type; Path = path; diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Responses/FileSearchResult.cs b/OpenAI/Packages/com.openai.unity/Runtime/Responses/FileSearchResult.cs index c031c74f..13c80334 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Responses/FileSearchResult.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Responses/FileSearchResult.cs @@ -15,7 +15,7 @@ internal FileSearchResult( [JsonProperty("file_id")] string fileId, [JsonProperty("text")] string text, [JsonProperty("file_name")] string fileName, - [JsonProperty("attributes")] IReadOnlyDictionary attributes, + [JsonProperty("attributes")] Dictionary attributes, [JsonProperty("score")] float? score) { FileId = fileId; diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Responses/FileSearchTool.cs b/OpenAI/Packages/com.openai.unity/Runtime/Responses/FileSearchTool.cs index afa098ed..fe8adcb7 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Responses/FileSearchTool.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Responses/FileSearchTool.cs @@ -36,10 +36,10 @@ public FileSearchTool(IEnumerable vectorStoreIds, int? maxNumberOfResult [JsonConstructor] internal FileSearchTool( [JsonProperty("type")] string type, - [JsonProperty("vector_store_ids")] IReadOnlyList vectorStoreIds, + [JsonProperty("vector_store_ids")] List vectorStoreIds, [JsonProperty("max_num_results", DefaultValueHandling = DefaultValueHandling.Ignore)] int? maxNumberOfResults, [JsonProperty("ranking_options", DefaultValueHandling = DefaultValueHandling.Ignore)] RankingOptions rankingOptions, - [JsonProperty("filters", DefaultValueHandling = DefaultValueHandling.Ignore)] IReadOnlyList filters) + [JsonProperty("filters", DefaultValueHandling = DefaultValueHandling.Ignore)] List filters) { Type = type; VectorStoreIds = vectorStoreIds; diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Responses/FileSearchToolCall.cs b/OpenAI/Packages/com.openai.unity/Runtime/Responses/FileSearchToolCall.cs index 63790a14..1b455287 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Responses/FileSearchToolCall.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Responses/FileSearchToolCall.cs @@ -19,8 +19,8 @@ internal FileSearchToolCall( [JsonProperty("type")] ResponseItemType type, [JsonProperty("object")] string @object, [JsonProperty("status")] ResponseStatus status, - [JsonProperty("queries")] IReadOnlyList queries, - [JsonProperty("results")] IReadOnlyList results) + [JsonProperty("queries")] List queries, + [JsonProperty("results")] List results) { Id = id; Type = type; diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Responses/KeyPressComputerAction.cs b/OpenAI/Packages/com.openai.unity/Runtime/Responses/KeyPressComputerAction.cs index 363d3e05..6535370f 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Responses/KeyPressComputerAction.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Responses/KeyPressComputerAction.cs @@ -18,7 +18,7 @@ public sealed class KeyPressComputerAction : IComputerAction [JsonConstructor] internal KeyPressComputerAction( [JsonProperty("type")] ComputerActionType type, - [JsonProperty("keys")] IReadOnlyList keys) + [JsonProperty("keys")] List keys) { Type = type; Keys = keys; diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Responses/LocalShellAction.cs b/OpenAI/Packages/com.openai.unity/Runtime/Responses/LocalShellAction.cs index 9ea69f94..f1a28783 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Responses/LocalShellAction.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Responses/LocalShellAction.cs @@ -14,8 +14,8 @@ public sealed class LocalShellAction [JsonConstructor] internal LocalShellAction( [JsonProperty("type")] string type, - [JsonProperty("command")] IReadOnlyList command, - [JsonProperty("env")] IReadOnlyDictionary environment, + [JsonProperty("command")] List command, + [JsonProperty("env")] Dictionary environment, [JsonProperty("timeout_ms")] int? timeoutMilliseconds, [JsonProperty("user")] string user, [JsonProperty("working_directory")] string workingDirectory) diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Responses/MCPListTools.cs b/OpenAI/Packages/com.openai.unity/Runtime/Responses/MCPListTools.cs index 684bcef2..09ab01ca 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Responses/MCPListTools.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Responses/MCPListTools.cs @@ -18,7 +18,7 @@ internal MCPListTools( [JsonProperty("status")] ResponseStatus status, [JsonProperty("server_label")] string serverLabel, [JsonProperty("error")] string error, - [JsonProperty("tools")] IReadOnlyList tools) + [JsonProperty("tools")] List tools) { Id = id; Type = type; diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Responses/MCPTool.cs b/OpenAI/Packages/com.openai.unity/Runtime/Responses/MCPTool.cs index b42a2578..fb4c220a 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Responses/MCPTool.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Responses/MCPTool.cs @@ -59,8 +59,8 @@ internal MCPTool( [JsonProperty("type")] string type, [JsonProperty("server_label")] string serverLabel, [JsonProperty("server_url")] string serverUrl, - [JsonProperty("allowed_tools")] IReadOnlyList allowedTools, - [JsonProperty("headers")] IReadOnlyDictionary headers, + [JsonProperty("allowed_tools")] List allowedTools, + [JsonProperty("headers")] Dictionary headers, [JsonProperty("require_approval")] object requireApproval) { Type = type; diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Responses/Message.cs b/OpenAI/Packages/com.openai.unity/Runtime/Responses/Message.cs index ab10f119..e9137c4d 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Responses/Message.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Responses/Message.cs @@ -29,7 +29,7 @@ internal Message( [JsonProperty("object")] string @object, [JsonProperty("status")] ResponseStatus status, [JsonProperty("role")] Role role, - [JsonProperty("content")] IReadOnlyList content) + [JsonProperty("content")] List content) { Id = id; Type = type; diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Responses/Prompt.cs b/OpenAI/Packages/com.openai.unity/Runtime/Responses/Prompt.cs index a2224ed1..9f2a7672 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Responses/Prompt.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Responses/Prompt.cs @@ -13,7 +13,7 @@ public sealed class Prompt [JsonConstructor] public Prompt( [JsonProperty("id")] string id, - [JsonProperty("variables")] IReadOnlyDictionary variables = null, + [JsonProperty("variables")] Dictionary variables = null, [JsonProperty("version")] string version = null) { Id = id; diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Responses/ReasoningContent.cs b/OpenAI/Packages/com.openai.unity/Runtime/Responses/ReasoningContent.cs new file mode 100644 index 00000000..f4f233dc --- /dev/null +++ b/OpenAI/Packages/com.openai.unity/Runtime/Responses/ReasoningContent.cs @@ -0,0 +1,53 @@ +// Licensed under the MIT License. See LICENSE in the project root for license information. + +using Newtonsoft.Json; +using UnityEngine.Scripting; +using Utilities.WebRequestRest.Interfaces; + +namespace OpenAI.Responses +{ + [Preserve] + public sealed class ReasoningContent : IServerSentEvent + { + [Preserve] + [JsonConstructor] + internal ReasoningContent( + [JsonProperty("type")] string type, + [JsonProperty("text")] string text) + { + Type = type; + Text = text; + } + + /// + /// The type of the reasoning text. Always reasoning_text. + /// + [Preserve] + [JsonProperty("type")] + public string Type { get; } + + /// + /// The reasoning text from the model. + /// + [Preserve] + [JsonProperty("text")] + public string Text { get; internal set; } + + [Preserve] + [JsonIgnore] + public string Delta { get; internal set; } + + [Preserve] + [JsonIgnore] + public string Object + => Type; + + [Preserve] + public string ToJsonString() + => JsonConvert.SerializeObject(this, OpenAIClient.JsonSerializationOptions); + + [Preserve] + public override string ToString() + => string.IsNullOrWhiteSpace(Text) ? Delta : Text; + } +} diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Responses/ReasoningContent.cs.meta b/OpenAI/Packages/com.openai.unity/Runtime/Responses/ReasoningContent.cs.meta new file mode 100644 index 00000000..5d201d2c --- /dev/null +++ b/OpenAI/Packages/com.openai.unity/Runtime/Responses/ReasoningContent.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 541be796729a7c54f918112e4b67fbb0 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {fileID: 2800000, guid: 84a7eb8fc6eba7540bf56cea8e12249c, type: 3} + userData: + assetBundleName: + assetBundleVariant: diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Responses/ReasoningItem.cs b/OpenAI/Packages/com.openai.unity/Runtime/Responses/ReasoningItem.cs index b0783228..cb8026dc 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Responses/ReasoningItem.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Responses/ReasoningItem.cs @@ -21,13 +21,16 @@ internal ReasoningItem( [JsonProperty("type")] ResponseItemType type, [JsonProperty("object")] string @object, [JsonProperty("status")] ResponseStatus status, - [JsonProperty("summary")] IReadOnlyList summary) + [JsonProperty("summary")] List summary, + [JsonProperty("content")] List content, + [JsonProperty("encrypted_content")] string encryptedContent) { Id = id; Type = type; Object = @object; Status = status; Summary = summary; + EncryptedContent = encryptedContent; } /// @@ -63,6 +66,47 @@ public IReadOnlyList Summary private set => summary = value?.ToList() ?? new(); } + private List content; + + /// + /// Reasoning text content. + /// + [Preserve] + [JsonProperty("content", DefaultValueHandling = DefaultValueHandling.Ignore)] + public IReadOnlyList Content + { + get => content; + private set => content = value?.ToList() ?? new(); + } + + /// + /// The encrypted content of the reasoning item - populated when a response is generated with reasoning.encrypted_content in the include parameter. + /// + [Preserve] + [JsonProperty("encrypted_content", DefaultValueHandling = DefaultValueHandling.Ignore)] + public string EncryptedContent { get; } + + [Preserve] + internal void InsertReasoningContent(ReasoningContent reasoningContent, int index) + { + if (reasoningContent == null) + { + throw new ArgumentNullException(nameof(reasoningContent)); + } + + content ??= new(); + + if (index > content.Count) + { + for (var i = content.Count; i < index; i++) + { + content.Add(null); + } + } + + content.Insert(index, reasoningContent); + } + [Preserve] internal void InsertSummary(ReasoningSummary item, int index) { diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Responses/ReasoningSummary.cs b/OpenAI/Packages/com.openai.unity/Runtime/Responses/ReasoningSummary.cs index 91286546..4f34ebd8 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Responses/ReasoningSummary.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Responses/ReasoningSummary.cs @@ -45,6 +45,6 @@ public string ToJsonString() [Preserve] public override string ToString() - => Text; + => string.IsNullOrWhiteSpace(Text) ? Delta : Text; } } diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Responses/Response.cs b/OpenAI/Packages/com.openai.unity/Runtime/Responses/Response.cs index 4dcb0f3a..5da85943 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Responses/Response.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Responses/Response.cs @@ -25,25 +25,30 @@ internal Response( [JsonProperty("object")] string @object, [JsonProperty("created_at")] int createdAtUnixSeconds, [JsonProperty("background")] bool? background = null, + [JsonProperty("conversation")] Conversation conversation = null, [JsonProperty("error")] Error error = null, [JsonProperty("incomplete_details")] IncompleteDetails incompleteDetails = null, - [JsonProperty("output")] IReadOnlyList output = null, + [JsonProperty("output")] List output = null, [JsonProperty("output_text")] string outputText = null, [JsonProperty("usage")] TokenUsage usage = null, [JsonProperty("parallel_tool_calls")] bool? parallelToolCalls = null, - [JsonProperty("instructions")][JsonConverter(typeof(StringOrObjectConverter>))] object instructions = null, + [JsonProperty("instructions")][JsonConverter(typeof(StringOrObjectConverter>))] object instructions = null, [JsonProperty("max_output_tokens")] int? maxOutputTokens = null, + [JsonProperty("max_tool_calls")] int? maxToolCalls = null, [JsonProperty("metadata")] Dictionary metadata = null, [JsonProperty("model")] string model = null, [JsonProperty("previous_response_id")] string previousResponseId = null, [JsonProperty("prompt")] Prompt prompt = null, + [JsonProperty("prompt_cache_key")] string promptCacheKey = null, [JsonProperty("reasoning")] Reasoning reasoning = null, + [JsonProperty("safety_identifier")] string safetyIdentifier = null, [JsonProperty("service_tier")] string serviceTier = null, [JsonProperty("status")] ResponseStatus status = 0, [JsonProperty("temperature")] double? temperature = null, [JsonProperty("text")] TextResponseFormatObject textResponseFormatObject = null, [JsonProperty("tool_choice")] object toolChoice = null, - [JsonProperty("tools")] IReadOnlyList tools = null, + [JsonProperty("tools")] List tools = null, + [JsonProperty("top_logprobs")] int? topLogProbs = null, [JsonProperty("top_p")] double? topP = null, [JsonProperty("truncation")] Truncation truncation = 0, [JsonProperty("user")] string user = null) @@ -52,6 +57,7 @@ internal Response( Object = @object; CreatedAtUnixSeconds = createdAtUnixSeconds; Background = background; + Conversation = conversation; Error = error; IncompleteDetails = incompleteDetails; Output = output; @@ -60,17 +66,21 @@ internal Response( ParallelToolCalls = parallelToolCalls; Instructions = instructions; MaxOutputTokens = maxOutputTokens; + MaxToolCalls = maxToolCalls; Metadata = metadata; Model = model; PreviousResponseId = previousResponseId; Prompt = prompt; + PromptCacheKey = promptCacheKey; Reasoning = reasoning; + SafetyIdentifier = safetyIdentifier; ServiceTier = serviceTier; Status = status; Temperature = temperature; TextResponseFormatObject = textResponseFormatObject; ToolChoice = toolChoice; Tools = tools; + TopLogProbs = topLogProbs; TopP = topP; Truncation = truncation; User = user; @@ -108,6 +118,14 @@ internal Response( [JsonProperty("background", DefaultValueHandling = DefaultValueHandling.Ignore)] public bool? Background { get; } + /// + /// The conversation that this response belongs to. + /// Input items and output items from this response are automatically added to this conversation. + /// + [Preserve] + [JsonProperty("conversation", DefaultValueHandling = DefaultValueHandling.Ignore)] + public Conversation Conversation { get; } + /// /// An error object returned when the model fails to generate a Response. /// @@ -164,6 +182,15 @@ public IReadOnlyList Output [JsonProperty("max_output_tokens", DefaultValueHandling = DefaultValueHandling.Ignore)] public int? MaxOutputTokens { get; } + /// + /// The maximum number of total calls to built-in tools that can be processed in a response. + /// This maximum number applies across all built-in tool calls, not per individual tool. + /// Any further attempts to call a tool by the model will be ignored. + /// + [Preserve] + [JsonProperty("max_tool_calls", DefaultValueHandling = DefaultValueHandling.Ignore)] + public int? MaxToolCalls { get; } + /// /// Set of 16 key-value pairs that can be attached to an object. /// This can be useful for storing additional information about the object in a structured format, @@ -199,6 +226,14 @@ public IReadOnlyList Output [JsonProperty("prompt", DefaultValueHandling = DefaultValueHandling.Ignore)] public Prompt Prompt { get; } + /// + /// Used by OpenAI to cache responses for similar requests to optimize your cache hit rates. + /// Replaces the user field. + /// + [Preserve] + [JsonProperty("prompt_cache_key", DefaultValueHandling = DefaultValueHandling.Ignore)] + public string PromptCacheKey { get; } + /// /// Configuration options for reasoning models. /// @@ -209,6 +244,15 @@ public IReadOnlyList Output [JsonProperty("reasoning", DefaultValueHandling = DefaultValueHandling.Ignore)] public Reasoning Reasoning { get; } + /// + /// A stable identifier used to help detect users of your application that may be violating OpenAI's usage policies. + /// The IDs should be a string that uniquely identifies each user. + /// We recommend hashing their username or email address, in order to avoid sending us any identifying information. + /// + [Preserve] + [JsonProperty("safety_identifier", DefaultValueHandling = DefaultValueHandling.Ignore)] + public string SafetyIdentifier { get; } + /// /// Specifies the latency tier to use for processing the request. This parameter is relevant for customers subscribed to the scale tier service:
/// - If set to 'auto', and the Project is Scale tier enabled, the system will utilize scale tier credits until they are exhausted.
@@ -266,6 +310,14 @@ public IReadOnlyList Output [JsonProperty("tools", DefaultValueHandling = DefaultValueHandling.Ignore)] public IReadOnlyList Tools { get; } + /// + /// An integer between 0 and 20 specifying the number of most likely tokens to return at each token position, + /// each with an associated log probability. + /// + [Preserve] + [JsonProperty("top_logprobs", DefaultValueHandling = DefaultValueHandling.Ignore)] + public int? TopLogProbs { get; } + /// /// An alternative to sampling with temperature, called nucleus sampling, /// where the model considers the results of the tokens with top_p probability mass. diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Responses/ResponseContentType.cs b/OpenAI/Packages/com.openai.unity/Runtime/Responses/ResponseContentType.cs index a0db741c..156a6a1f 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Responses/ResponseContentType.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Responses/ResponseContentType.cs @@ -19,6 +19,6 @@ public enum ResponseContentType [EnumMember(Value = "input_file")] InputFile, [EnumMember(Value = "refusal")] - Refusal, + Refusal } } diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Responses/ResponsesEndpoint.cs b/OpenAI/Packages/com.openai.unity/Runtime/Responses/ResponsesEndpoint.cs index 2dd6dbd2..ad8db69c 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Responses/ResponsesEndpoint.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Responses/ResponsesEndpoint.cs @@ -152,6 +152,7 @@ private async Task StreamResponseAsync(string endpoint, string payload switch (@event) { case "response.created": + case "response.queued": case "response.in_progress": case "response.completed": case "response.failed": @@ -292,6 +293,20 @@ private async Task StreamResponseAsync(string endpoint, string payload serverSentEvent = summaryItem; } + break; + case "response.reasoning_text.delta": + case "response.reasoning_text.done": + var reasoningContentIndex = @object["content_index"]!.Value(); + reasoningItem = (ReasoningItem)response!.Output[outputIndex!.Value]; + var reasoningContentItem = reasoningItem.Content[reasoningContentIndex]; + + if (!string.IsNullOrWhiteSpace(text)) + { + reasoningContentItem.Text = text; + } + + reasoningContentItem.Delta = !string.IsNullOrWhiteSpace(delta) ? delta : null; + serverSentEvent = reasoningContentItem; break; case "response.reasoning_summary_text.delta": case "response.reasoning_summary_text.done": @@ -310,17 +325,34 @@ private async Task StreamResponseAsync(string endpoint, string payload case "error": serverSentEvent = sseResponse.Deserialize(client); break; - case "response.code_interpreter_call.code.delta": - case "response.code_interpreter_call.code.done": - case "response.code_interpreter_call.completed": - case "response.code_interpreter_call.in_progress": + // TODO - implement handling for these events: case "response.code_interpreter_call.interpreting": - case "response.file_search_call.completed": + case "response.code_interpreter_call.in_progress": + case "response.code_interpreter_call.completed": + case "response.code_interpreter_call_code.delta": + case "response.code_interpreter_call_code.done": + case "response.custom_tool_call_input.delta": + case "response.custom_tool_call_input.done": case "response.file_search_call.in_progress": case "response.file_search_call.searching": - case "response.web_search_call.completed": + case "response.file_search_call.completed": + case "response.function_call_arguments.delta": + case "response.function_call_arguments.done": + case "response.image_generation_call.in_progress": + case "response.image_generation_call.generating": + case "response.image_generation_call.partial_image": + case "response.image_generation_call.completed": + case "response.mcp_call_arguments.delta": + case "response.mcp_call_arguments.done": + case "response.mcp_call.in_progress": + case "response.mcp_call.completed": + case "response.mcp_call.failed": + case "response.mcp_list_tools.in_progress": + case "response.mcp_list_tools.completed": + case "response.mcp_list_tools.failed": case "response.web_search_call.in_progress": case "response.web_search_call.searching": + case "response.web_search_call.completed": default: // if not properly handled raise it up to caller to deal with it themselves. serverSentEvent = ssEvent; diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Responses/TextContent.cs b/OpenAI/Packages/com.openai.unity/Runtime/Responses/TextContent.cs index 9bcab9d0..11684639 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Responses/TextContent.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Responses/TextContent.cs @@ -19,8 +19,8 @@ public sealed class TextContent : BaseResponse, IResponseContent internal TextContent( [JsonProperty("type")] ResponseContentType type, [JsonProperty("text")] string text, - [JsonProperty("annotations")] IReadOnlyList annotations, - IReadOnlyList logProbs) + [JsonProperty("annotations")] List annotations, + [JsonProperty("log_probs")] List logProbs) { Type = type; Text = text; diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Threads/CodeInterpreterOutputs.cs b/OpenAI/Packages/com.openai.unity/Runtime/Threads/CodeInterpreterOutputs.cs index 14359b7e..85b49f0b 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Threads/CodeInterpreterOutputs.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Threads/CodeInterpreterOutputs.cs @@ -17,7 +17,7 @@ internal CodeInterpreterOutputs( [JsonProperty("type")] CodeInterpreterOutputType type, [JsonProperty("logs")] string logs, [JsonProperty("image")] ImageFile image, - [JsonProperty("files")] IReadOnlyList files) + [JsonProperty("files")] List files) { Index = index; Type = type; diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Threads/Message.cs b/OpenAI/Packages/com.openai.unity/Runtime/Threads/Message.cs index 27df2911..72bff25f 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Threads/Message.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Threads/Message.cs @@ -36,7 +36,7 @@ public Message( Role role = Role.User, IEnumerable attachments = null, IReadOnlyDictionary metadata = null) - : this(new List { new(content) }, role, attachments, metadata) + : this(new List { new(content) }, role, attachments, (Dictionary)metadata) { } @@ -63,7 +63,7 @@ public Message( [JsonProperty("content")] IEnumerable content, [JsonProperty("role")] Role role = Role.User, [JsonProperty("attachments")] IEnumerable attachments = null, - [JsonProperty("metadata")] IReadOnlyDictionary metadata = null) + [JsonProperty("metadata")] Dictionary metadata = null) { Content = content?.ToList(); Role = role; diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Threads/MessageDelta.cs b/OpenAI/Packages/com.openai.unity/Runtime/Threads/MessageDelta.cs index ff8559e9..df8dbcb5 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Threads/MessageDelta.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Threads/MessageDelta.cs @@ -14,7 +14,7 @@ public sealed class MessageDelta [JsonConstructor] internal MessageDelta( [JsonProperty("role")] Role role, - [JsonProperty("content")] IReadOnlyList content) + [JsonProperty("content")] List content) { Role = role; Content = content; diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Threads/MessageResponse.cs b/OpenAI/Packages/com.openai.unity/Runtime/Threads/MessageResponse.cs index d21f2f29..92bfc7ff 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Threads/MessageResponse.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Threads/MessageResponse.cs @@ -37,7 +37,7 @@ internal MessageResponse( [JsonProperty("content")] List content, [JsonProperty("assistant_id")] string assistantId, [JsonProperty("run_id")] string runId, - [JsonProperty("attachments")] IReadOnlyList attachments, + [JsonProperty("attachments")] List attachments, [JsonProperty("metadata")] Dictionary metadata) { Id = id; @@ -184,7 +184,7 @@ public DateTime? IncompleteAt [Preserve] public static implicit operator Message(MessageResponse response) - => new(response.Content, response.Role, response.Attachments, response.Metadata); + => new(response.Content, response.Role, response.Attachments, (Dictionary)response.Metadata); [Preserve] public override string ToString() => Id; diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Threads/SubmitToolOutputs.cs b/OpenAI/Packages/com.openai.unity/Runtime/Threads/SubmitToolOutputs.cs index ab5f5ad3..939c7898 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Threads/SubmitToolOutputs.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Threads/SubmitToolOutputs.cs @@ -11,7 +11,7 @@ public sealed class SubmitToolOutputs { [Preserve] [JsonConstructor] - internal SubmitToolOutputs([JsonProperty("tool_calls")] IReadOnlyList toolCalls) + internal SubmitToolOutputs([JsonProperty("tool_calls")] List toolCalls) { ToolCalls = toolCalls; } diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Threads/ToolCall.cs b/OpenAI/Packages/com.openai.unity/Runtime/Threads/ToolCall.cs index e5c1d37b..517842c4 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Threads/ToolCall.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Threads/ToolCall.cs @@ -24,7 +24,7 @@ internal ToolCall( [JsonProperty("type")] string type, [JsonProperty("function")] FunctionCall functionCall, [JsonProperty("code_interpreter")] CodeInterpreter codeInterpreter, - [JsonProperty("file_search")] IReadOnlyDictionary fileSearch) + [JsonProperty("file_search")] Dictionary fileSearch) { Index = index; Id = id; From d37dd9100ad0b540055aae71fcaa3f659fa6d56d Mon Sep 17 00:00:00 2001 From: Stephen Hodgson Date: Mon, 29 Sep 2025 22:43:41 -0400 Subject: [PATCH 08/22] preserve --- .../Packages/com.openai.unity/Runtime/Responses/Conversation.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Responses/Conversation.cs b/OpenAI/Packages/com.openai.unity/Runtime/Responses/Conversation.cs index 40af95c3..24176811 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Responses/Conversation.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Responses/Conversation.cs @@ -33,6 +33,7 @@ internal Conversation( /// /// The datetime at which the conversation was created. /// + [Preserve] [JsonIgnore] public DateTime CreatedAt => DateTimeOffset.FromUnixTimeSeconds(CreatedAtUnitTimeSeconds).UtcDateTime; From 3e9fc34dd043dbe899afa498d509aea929e40ac4 Mon Sep 17 00:00:00 2001 From: Stephen Hodgson Date: Mon, 29 Sep 2025 23:12:35 -0400 Subject: [PATCH 09/22] fix reasoning content impl added missing function call impl for responses api streaming --- .../Runtime/ConversationsEndpoint.cs | 5 +-- .../Extensions/ResponseContentConverter.cs | 1 + .../Runtime/Responses/ReasoningContent.cs | 14 ++---- .../Runtime/Responses/ResponseContentType.cs | 4 +- .../Runtime/Responses/ResponsesEndpoint.cs | 44 ++++++++++++------- 5 files changed, 38 insertions(+), 30 deletions(-) diff --git a/OpenAI/Packages/com.openai.unity/Runtime/ConversationsEndpoint.cs b/OpenAI/Packages/com.openai.unity/Runtime/ConversationsEndpoint.cs index 6de3bfed..c782b64a 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/ConversationsEndpoint.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/ConversationsEndpoint.cs @@ -6,7 +6,6 @@ using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; -using UnityEngine.TextCore.Text; using Utilities.WebRequestRest; namespace OpenAI.Responses @@ -142,7 +141,7 @@ public async Task> CreateConversationItemsAsync(stri var payload = JsonConvert.SerializeObject(new { items }, OpenAIClient.JsonSerializationOptions); Dictionary query = null; - if (include != null) + if (include is { Length: > 0 }) { query = new Dictionary { @@ -177,7 +176,7 @@ public async Task GetConversationItemAsync(string conversationId, Dictionary query = null; - if (include != null) + if (include is { Length: > 0 }) { query = new Dictionary { diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Extensions/ResponseContentConverter.cs b/OpenAI/Packages/com.openai.unity/Runtime/Extensions/ResponseContentConverter.cs index 74be791a..2d6a9b41 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Extensions/ResponseContentConverter.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Extensions/ResponseContentConverter.cs @@ -28,6 +28,7 @@ public override object ReadJson(JsonReader reader, Type objectType, object exist "input_image" => jObject.ToObject(serializer), "input_file" => jObject.ToObject(serializer), "refusal" => jObject.ToObject(serializer), + "reasoning_text" => jObject.ToObject(serializer), _ => throw new NotImplementedException($"Unknown response content type: {type}") }; } diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Responses/ReasoningContent.cs b/OpenAI/Packages/com.openai.unity/Runtime/Responses/ReasoningContent.cs index f4f233dc..152c3d73 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Responses/ReasoningContent.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Responses/ReasoningContent.cs @@ -2,17 +2,16 @@ using Newtonsoft.Json; using UnityEngine.Scripting; -using Utilities.WebRequestRest.Interfaces; namespace OpenAI.Responses { [Preserve] - public sealed class ReasoningContent : IServerSentEvent + public sealed class ReasoningContent : BaseResponse, IResponseContent { [Preserve] [JsonConstructor] internal ReasoningContent( - [JsonProperty("type")] string type, + [JsonProperty("type")] ResponseContentType type, [JsonProperty("text")] string text) { Type = type; @@ -24,7 +23,7 @@ internal ReasoningContent( /// [Preserve] [JsonProperty("type")] - public string Type { get; } + public ResponseContentType Type { get; } /// /// The reasoning text from the model. @@ -39,12 +38,7 @@ internal ReasoningContent( [Preserve] [JsonIgnore] - public string Object - => Type; - - [Preserve] - public string ToJsonString() - => JsonConvert.SerializeObject(this, OpenAIClient.JsonSerializationOptions); + public string Object => Type.ToString(); [Preserve] public override string ToString() diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Responses/ResponseContentType.cs b/OpenAI/Packages/com.openai.unity/Runtime/Responses/ResponseContentType.cs index 156a6a1f..4a6c7326 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Responses/ResponseContentType.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Responses/ResponseContentType.cs @@ -19,6 +19,8 @@ public enum ResponseContentType [EnumMember(Value = "input_file")] InputFile, [EnumMember(Value = "refusal")] - Refusal + Refusal, + [EnumMember(Value = "reasoning_text")] + ReasoningText } } diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Responses/ResponsesEndpoint.cs b/OpenAI/Packages/com.openai.unity/Runtime/Responses/ResponsesEndpoint.cs index ad8db69c..a69495c5 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Responses/ResponsesEndpoint.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Responses/ResponsesEndpoint.cs @@ -204,6 +204,20 @@ private async Task StreamResponseAsync(string endpoint, string payload serverSentEvent = item; } break; + case "response.function_call_arguments.delta": + case "response.function_call_arguments.done": + var functionToolCall = (FunctionToolCall)response!.Output[outputIndex!.Value]; + + if (functionToolCall.Id != itemId) + { + throw new InvalidOperationException($"FunctionToolCall ID mismatch! Expected: {functionToolCall.Id}, got: {itemId}"); + } + + if (!string.IsNullOrWhiteSpace(delta)) + { + functionToolCall.Delta = delta; + } + break; case "response.audio.delta": case "response.audio.done": case "response.audio.transcript.delta": @@ -213,6 +227,8 @@ private async Task StreamResponseAsync(string endpoint, string payload case "response.output_text.done": case "response.refusal.delta": case "response.refusal.done": + case "response.reasoning_summary_text.delta": + case "response.reasoning_summary_text.done": messageItem = (Message)response!.Output[outputIndex!.Value]; if (messageItem.Id != itemId) @@ -278,6 +294,18 @@ private async Task StreamResponseAsync(string endpoint, string payload } serverSentEvent = refusalContent; + break; + case ReasoningContent reasoningContent: + if (!string.IsNullOrWhiteSpace(text)) + { + reasoningContent.Text = text; + } + + if (!string.IsNullOrWhiteSpace(delta)) + { + reasoningContent.Delta = delta; + } + break; } break; @@ -308,20 +336,6 @@ private async Task StreamResponseAsync(string endpoint, string payload reasoningContentItem.Delta = !string.IsNullOrWhiteSpace(delta) ? delta : null; serverSentEvent = reasoningContentItem; break; - case "response.reasoning_summary_text.delta": - case "response.reasoning_summary_text.done": - summaryIndex = @object["summary_index"]!.Value(); - reasoningItem = (ReasoningItem)response!.Output[outputIndex!.Value]; - summaryItem = reasoningItem.Summary[summaryIndex]; - - if (!string.IsNullOrWhiteSpace(text)) - { - summaryItem.Text = text; - } - - summaryItem.Delta = !string.IsNullOrWhiteSpace(delta) ? delta : null; - serverSentEvent = summaryItem; - break; case "error": serverSentEvent = sseResponse.Deserialize(client); break; @@ -336,8 +350,6 @@ private async Task StreamResponseAsync(string endpoint, string payload case "response.file_search_call.in_progress": case "response.file_search_call.searching": case "response.file_search_call.completed": - case "response.function_call_arguments.delta": - case "response.function_call_arguments.done": case "response.image_generation_call.in_progress": case "response.image_generation_call.generating": case "response.image_generation_call.partial_image": From 634a17005a9128eb5a1c25b735d5d90adcc2c77a Mon Sep 17 00:00:00 2001 From: Stephen Hodgson Date: Mon, 29 Sep 2025 23:40:10 -0400 Subject: [PATCH 10/22] update response streaming debug --- .../Runtime/Responses/ResponsesEndpoint.cs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Responses/ResponsesEndpoint.cs b/OpenAI/Packages/com.openai.unity/Runtime/Responses/ResponsesEndpoint.cs index a69495c5..6cdbace7 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Responses/ResponsesEndpoint.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Responses/ResponsesEndpoint.cs @@ -132,13 +132,14 @@ private async Task StreamResponseAsync(string endpoint, string payload var streamResponse = await Rest.PostAsync(endpoint, payload, async (sseResponse, ssEvent) => { + IServerSentEvent serverSentEvent = null; + var @event = ssEvent.Value.Value(); + if (EnableDebug) { - Debug.Log($"{ssEvent.ToJsonString()}"); + Debug.Log($"\"{@event}\": {ssEvent.ToJsonString()}"); } - IServerSentEvent serverSentEvent = null; - var @event = ssEvent.Value.Value(); var @object = ssEvent.Data ?? ssEvent.Value; var text = @object["text"]?.Value(); var delta = @object["delta"]?.Value(); From 4fd799dee4aec6ad720ae647e3a086e5c660851a Mon Sep 17 00:00:00 2001 From: Stephen Hodgson Date: Tue, 30 Sep 2025 08:18:23 -0400 Subject: [PATCH 11/22] see event --- .../com.openai.unity/Runtime/Responses/ResponsesEndpoint.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Responses/ResponsesEndpoint.cs b/OpenAI/Packages/com.openai.unity/Runtime/Responses/ResponsesEndpoint.cs index 6cdbace7..6553d1d6 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Responses/ResponsesEndpoint.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Responses/ResponsesEndpoint.cs @@ -137,7 +137,7 @@ private async Task StreamResponseAsync(string endpoint, string payload if (EnableDebug) { - Debug.Log($"\"{@event}\": {ssEvent.ToJsonString()}"); + Debug.Log(ssEvent.ToJsonString()); } var @object = ssEvent.Data ?? ssEvent.Value; From 8b05965a3007615f8ab9f45aa85675a546ec5b3e Mon Sep 17 00:00:00 2001 From: Stephen Hodgson Date: Tue, 30 Sep 2025 09:11:17 -0400 Subject: [PATCH 12/22] - fix MCPTool serialzations - added CustomToolCall response items --- .../Extensions/ResponseItemConverter.cs | 2 + .../Runtime/Responses/CustomToolCall.cs | 56 ++++++++++ .../Runtime/Responses/MCPServerTool.cs | 28 ++++- .../Runtime/Responses/MCPTool.cs | 104 ++++++++++++++++-- .../Runtime/Responses/MCPToolCall.cs | 15 ++- .../Responses/MCPToolRequireApproval.cs | 14 +++ .../Runtime/Responses/ResponseItemType.cs | 4 + 7 files changed, 205 insertions(+), 18 deletions(-) create mode 100644 OpenAI/Packages/com.openai.unity/Runtime/Responses/CustomToolCall.cs create mode 100644 OpenAI/Packages/com.openai.unity/Runtime/Responses/MCPToolRequireApproval.cs diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Extensions/ResponseItemConverter.cs b/OpenAI/Packages/com.openai.unity/Runtime/Extensions/ResponseItemConverter.cs index 195cb675..e4828a4d 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Extensions/ResponseItemConverter.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Extensions/ResponseItemConverter.cs @@ -25,6 +25,8 @@ public override object ReadJson(JsonReader reader, Type objectType, object exist "computer_call_output" => jObject.ToObject(serializer), "function_call" => jObject.ToObject(serializer), "function_call_output" => jObject.ToObject(serializer), + "custom_tool_call" => jObject.ToObject(serializer), + "custom_tool_call_output" => jObject.ToObject(serializer), "image_generation_call" => jObject.ToObject(serializer), "local_shell_call" => jObject.ToObject(serializer), "local_shell_call_output" => jObject.ToObject(serializer), diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Responses/CustomToolCall.cs b/OpenAI/Packages/com.openai.unity/Runtime/Responses/CustomToolCall.cs new file mode 100644 index 00000000..96791803 --- /dev/null +++ b/OpenAI/Packages/com.openai.unity/Runtime/Responses/CustomToolCall.cs @@ -0,0 +1,56 @@ +// Licensed under the MIT License. See LICENSE in the project root for license information. + +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using UnityEngine.Scripting; + +namespace OpenAI.Responses +{ + /// + /// A call to a custom tool created by the model. + /// + [Preserve] + public sealed class CustomToolCall : BaseResponse, IResponseItem, IToolCall + { + /// + [Preserve] + [JsonProperty("id", DefaultValueHandling = DefaultValueHandling.Ignore)] + public string Id { get; } + + /// + [Preserve] + [JsonProperty("type", DefaultValueHandling = DefaultValueHandling.Include)] + public ResponseItemType Type { get; } + + /// + [Preserve] + [JsonProperty("object", DefaultValueHandling = DefaultValueHandling.Ignore)] + public string Object { get; } + + /// + [Preserve] + [JsonProperty("status", DefaultValueHandling = DefaultValueHandling.Ignore)] + public ResponseStatus Status { get; } + + /// + /// An identifier used to map this custom tool call to a tool call output. + /// + [Preserve] + [JsonProperty("call_id")] + public string CallId { get; } + + /// + /// The name of the custom tool being called. + /// + [Preserve] + [JsonProperty("name")] + public string Name { get; } + + /// + /// The input for the custom tool call generated by the model. + /// + [Preserve] + [JsonProperty("input")] + public JToken Arguments { get; } + } +} diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Responses/MCPServerTool.cs b/OpenAI/Packages/com.openai.unity/Runtime/Responses/MCPServerTool.cs index c41d719e..fcb3f177 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Responses/MCPServerTool.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Responses/MCPServerTool.cs @@ -1,10 +1,14 @@ // Licensed under the MIT License. See LICENSE in the project root for license information. using Newtonsoft.Json; +using Newtonsoft.Json.Linq; using UnityEngine.Scripting; namespace OpenAI.Responses { + /// + /// A tool available on an MCP Server + /// [Preserve] public sealed class MCPServerTool { @@ -13,8 +17,8 @@ public sealed class MCPServerTool internal MCPServerTool( [JsonProperty("name")] string name, [JsonProperty("description")] string description, - [JsonProperty("input_schema")] string inputSchema, - [JsonProperty("annotations")] object annotations) + [JsonProperty("input_schema")] JToken inputSchema, + [JsonProperty("annotations")] JToken annotations) { Name = name; Description = description; @@ -22,20 +26,32 @@ internal MCPServerTool( Annotations = annotations; } + /// + /// The name of the tool. + /// [Preserve] [JsonProperty("name")] public string Name { get; } + /// + /// The description of the tool. + /// [Preserve] - [JsonProperty("description")] + [JsonProperty("description", DefaultValueHandling = DefaultValueHandling.Ignore)] public string Description { get; } + /// + /// The JSON schema describing the tool's input. + /// [Preserve] [JsonProperty("input_schema")] - public string InputSchema { get; } + public JToken InputSchema { get; } + /// + /// Additional annotations about the tool. + /// [Preserve] - [JsonProperty("annotations")] - public object Annotations { get; } + [JsonProperty("annotations", DefaultValueHandling = DefaultValueHandling.Ignore)] + public JToken Annotations { get; } } } diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Responses/MCPTool.cs b/OpenAI/Packages/com.openai.unity/Runtime/Responses/MCPTool.cs index fb4c220a..5b4e935b 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Responses/MCPTool.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Responses/MCPTool.cs @@ -9,6 +9,7 @@ namespace OpenAI.Responses { /// /// Give the model access to additional tools via remote Model Context Protocol (MCP) servers. + /// /// [Preserve] public sealed class MCPTool : ITool @@ -20,10 +21,21 @@ public sealed class MCPTool : ITool public MCPTool( string serverLabel, string serverUrl, + string connectorId, + string authorization, + string serverDescription, IReadOnlyList allowedTools, IReadOnlyDictionary headers, - string requireApproval) - : this(serverLabel, serverUrl, allowedTools, headers, (object)requireApproval) + MCPToolRequireApproval requireApproval) + : this( + serverLabel: serverLabel, + serverUrl: serverUrl, + connectorId: connectorId, + authorization: authorization, + serverDescription: serverDescription, + allowedTools: allowedTools, + headers: headers, + requireApproval: requireApproval.ToString().ToLower()) { } @@ -31,23 +43,40 @@ public MCPTool( public MCPTool( string serverLabel, string serverUrl, + string connectorId, + string authorization, + string serverDescription, IReadOnlyList allowedTools, IReadOnlyDictionary headers, MCPApprovalFilter requireApproval) - : this(serverLabel, serverUrl, allowedTools, headers, (object)requireApproval) + : this( + serverLabel: serverLabel, + serverUrl: serverUrl, + connectorId: connectorId, + authorization: authorization, + serverDescription: serverDescription, + allowedTools: allowedTools, + headers: headers, + requireApproval: (object)requireApproval) { } [Preserve] public MCPTool( string serverLabel, - string serverUrl, + string serverUrl = null, + string connectorId = null, + string authorization = null, + string serverDescription = null, IReadOnlyList allowedTools = null, IReadOnlyDictionary headers = null, object requireApproval = null) { ServerLabel = serverLabel; ServerUrl = serverUrl; + ConnectorId = connectorId; + Authorization = authorization; + ServerDescription = serverDescription; AllowedTools = allowedTools; Headers = headers ?? new Dictionary(); RequireApproval = requireApproval; @@ -59,18 +88,27 @@ internal MCPTool( [JsonProperty("type")] string type, [JsonProperty("server_label")] string serverLabel, [JsonProperty("server_url")] string serverUrl, - [JsonProperty("allowed_tools")] List allowedTools, + [JsonProperty("connector_id")] string connectorId, + [JsonProperty("authorization")] string authorization, + [JsonProperty("server_description")] string serverDescription, + [JsonProperty("allowed_tools")] List allowedTools, [JsonProperty("headers")] Dictionary headers, [JsonProperty("require_approval")] object requireApproval) { Type = type; ServerLabel = serverLabel; ServerUrl = serverUrl; + ConnectorId = connectorId; + Authorization = authorization; + ServerDescription = serverDescription; AllowedTools = allowedTools; Headers = headers; RequireApproval = requireApproval; } + /// + /// The type of the MCP tool. Always `mcp`. + /// [Preserve] [JsonProperty("type")] public string Type { get; } = "mcp"; @@ -79,9 +117,61 @@ internal MCPTool( /// A label for this MCP server, used to identify it in tool calls. /// [Preserve] - [JsonProperty("server_label", DefaultValueHandling = DefaultValueHandling.Ignore)] + [JsonProperty("server_label")] public string ServerLabel { get; } + /// + /// Identifier for service connectors, like those available in ChatGPT. One of + /// or must be provided. Learn more about service + /// connectors .
+ /// Currently supported `connector_id` values are:
+ /// + /// + /// Dropbox: connector_dropbox + /// + /// + /// Gmail: connector_gmail + /// + /// + /// Google Calendar: connector_googlecalendar + /// + /// + /// Google Drive: connector_googledrive + /// + /// + /// Microsoft Teams: connector_microsoftteams + /// + /// + /// Outlook Calendar: connector_outlookcalendar + /// + /// + /// Outlook Email: connector_outlookemail + /// + /// + /// SharePoint: connector_sharepoint + /// + /// + /// + [Preserve] + [JsonProperty("connector_id", DefaultValueHandling = DefaultValueHandling.Ignore)] + public string ConnectorId { get; } + + /// + /// An OAuth access token that can be used with a remote MCP server, either + /// with a custom MCP server URL or a service connector. Your application + /// must handle the OAuth authorization flow and provide the token here. + /// + [Preserve] + [JsonProperty("authorization", DefaultValueHandling = DefaultValueHandling.Ignore)] + public string Authorization { get; } + + /// + /// Optional description of the MCP server, used to provide more context. + /// + [Preserve] + [JsonProperty("server_description", DefaultValueHandling = DefaultValueHandling.Ignore)] + public string ServerDescription { get; } + /// /// The URL for the MCP server. /// @@ -94,7 +184,7 @@ internal MCPTool( /// [Preserve] [JsonProperty("allowed_tools", DefaultValueHandling = DefaultValueHandling.Ignore)] - public IReadOnlyList AllowedTools { get; } + public IReadOnlyList AllowedTools { get; } /// /// Optional HTTP headers to send to the MCP server. Use for authentication or other purposes. diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Responses/MCPToolCall.cs b/OpenAI/Packages/com.openai.unity/Runtime/Responses/MCPToolCall.cs index c197aca3..8bbd650e 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Responses/MCPToolCall.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Responses/MCPToolCall.cs @@ -1,10 +1,14 @@ // Licensed under the MIT License. See LICENSE in the project root for license information. using Newtonsoft.Json; +using Newtonsoft.Json.Linq; using UnityEngine.Scripting; namespace OpenAI.Responses { + /// + /// An invocation of a tool on an MCP server. + /// [Preserve] public sealed class MCPToolCall : BaseResponse, IResponseItem { @@ -17,8 +21,9 @@ internal MCPToolCall( [JsonProperty("status")] ResponseStatus status, [JsonProperty("name")] string name, [JsonProperty("server_label")] string serverLabel, - [JsonProperty("arguments")] string arguments, - [JsonProperty("output")] string output) + [JsonProperty("arguments")] JToken arguments, + [JsonProperty("output")] string output, + [JsonProperty("error")] string error) { Id = id; Type = type; @@ -61,15 +66,15 @@ internal MCPToolCall( /// The label of the MCP server running the tool. /// [Preserve] - [JsonProperty("server_label", DefaultValueHandling = DefaultValueHandling.Ignore)] + [JsonProperty("server_label")] public string ServerLabel { get; } /// /// A JSON string of the arguments to pass to the function. /// [Preserve] - [JsonProperty("arguments", DefaultValueHandling = DefaultValueHandling.Ignore)] - public string Arguments { get; } + [JsonProperty("arguments")] + public JToken Arguments { get; } /// /// The output from the tool call. diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Responses/MCPToolRequireApproval.cs b/OpenAI/Packages/com.openai.unity/Runtime/Responses/MCPToolRequireApproval.cs new file mode 100644 index 00000000..80ad7815 --- /dev/null +++ b/OpenAI/Packages/com.openai.unity/Runtime/Responses/MCPToolRequireApproval.cs @@ -0,0 +1,14 @@ +// Licensed under the MIT License. See LICENSE in the project root for license information. + +using System.Runtime.Serialization; + +namespace OpenAI.Responses +{ + public enum MCPToolRequireApproval + { + [EnumMember(Value = "never")] + Never, + [EnumMember(Value = "always")] + Always + } +} diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Responses/ResponseItemType.cs b/OpenAI/Packages/com.openai.unity/Runtime/Responses/ResponseItemType.cs index ba22ae87..0b10f98c 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Responses/ResponseItemType.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Responses/ResponseItemType.cs @@ -40,5 +40,9 @@ public enum ResponseItemType McpListTools, [EnumMember(Value = "item_reference")] ItemReference, + [EnumMember(Value = "custom_tool_call")] + CustomToolCall, + [EnumMember(Value = "custom_tool_call_output")] + CustomToolCallOutput } } From b547c9927e44e53bd88f0662a580abf773f3cb15 Mon Sep 17 00:00:00 2001 From: Stephen Hodgson Date: Tue, 30 Sep 2025 09:35:13 -0400 Subject: [PATCH 13/22] add custom tool call output --- .../Extensions/ResponseItemConverter.cs | 2 +- .../Runtime/Responses/CustomToolCall.cs | 31 +++++++- .../Runtime/Responses/CustomToolCallOutput.cs | 78 +++++++++++++++++++ 3 files changed, 109 insertions(+), 2 deletions(-) create mode 100644 OpenAI/Packages/com.openai.unity/Runtime/Responses/CustomToolCallOutput.cs diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Extensions/ResponseItemConverter.cs b/OpenAI/Packages/com.openai.unity/Runtime/Extensions/ResponseItemConverter.cs index e4828a4d..5727bc96 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Extensions/ResponseItemConverter.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Extensions/ResponseItemConverter.cs @@ -26,7 +26,7 @@ public override object ReadJson(JsonReader reader, Type objectType, object exist "function_call" => jObject.ToObject(serializer), "function_call_output" => jObject.ToObject(serializer), "custom_tool_call" => jObject.ToObject(serializer), - "custom_tool_call_output" => jObject.ToObject(serializer), + "custom_tool_call_output" => jObject.ToObject(serializer), "image_generation_call" => jObject.ToObject(serializer), "local_shell_call" => jObject.ToObject(serializer), "local_shell_call_output" => jObject.ToObject(serializer), diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Responses/CustomToolCall.cs b/OpenAI/Packages/com.openai.unity/Runtime/Responses/CustomToolCall.cs index 96791803..1bc6a1f2 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Responses/CustomToolCall.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Responses/CustomToolCall.cs @@ -12,6 +12,32 @@ namespace OpenAI.Responses [Preserve] public sealed class CustomToolCall : BaseResponse, IResponseItem, IToolCall { + [Preserve] + public CustomToolCall(string callId, string name, string input) + { + CallId = callId; + Name = name; + Input = input; + } + + [JsonConstructor] + internal CustomToolCall( + [JsonProperty("id")] string id, + [JsonProperty("object")] string @object, + [JsonProperty("status")] ResponseStatus status, + [JsonProperty("call_id")] string callId, + [JsonProperty("name")] string name, + [JsonProperty("input")] string input) + { + Id = id; + Object = @object; + Status = status; + CallId = callId; + Name = name; + Input = input; + Type = ResponseItemType.CustomToolCall; + } + /// [Preserve] [JsonProperty("id", DefaultValueHandling = DefaultValueHandling.Ignore)] @@ -46,11 +72,14 @@ public sealed class CustomToolCall : BaseResponse, IResponseItem, IToolCall [JsonProperty("name")] public string Name { get; } + [JsonIgnore] + public JToken Arguments => null; + /// /// The input for the custom tool call generated by the model. /// [Preserve] [JsonProperty("input")] - public JToken Arguments { get; } + public string Input { get; } } } diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Responses/CustomToolCallOutput.cs b/OpenAI/Packages/com.openai.unity/Runtime/Responses/CustomToolCallOutput.cs new file mode 100644 index 00000000..5600d779 --- /dev/null +++ b/OpenAI/Packages/com.openai.unity/Runtime/Responses/CustomToolCallOutput.cs @@ -0,0 +1,78 @@ +// Licensed under the MIT License. See LICENSE in the project root for license information. + +using Newtonsoft.Json; +using UnityEngine.Scripting; + +namespace OpenAI.Responses +{ + public sealed class CustomToolCallOutput : BaseResponse, IResponseItem + { + [Preserve] + public CustomToolCallOutput(CustomToolCall toolCall, string output) + { + CallId = toolCall.CallId; + Output = output; + } + + [Preserve] + public CustomToolCallOutput(string callId, string output) + { + CallId = callId; + Output = output; + } + + [Preserve] + [JsonConstructor] + internal CustomToolCallOutput( + [JsonProperty("id")] string id, + [JsonProperty("object")] string @object, + [JsonProperty("status")] ResponseStatus status, + [JsonProperty("call_id")] string callId, + [JsonProperty("output")] string output) + { + Id = id; + Object = @object; + Status = status; + CallId = callId; + Output = output; + } + + /// + [Preserve] + [JsonProperty("id", DefaultValueHandling = DefaultValueHandling.Ignore)] + public string Id { get; } + + /// + [Preserve] + [JsonProperty("type", DefaultValueHandling = DefaultValueHandling.Include)] + public ResponseItemType Type { get; } = ResponseItemType.FunctionCallOutput; + + /// + [Preserve] + [JsonProperty("object", DefaultValueHandling = DefaultValueHandling.Ignore)] + public string Object { get; } + + /// + [Preserve] + [JsonProperty("status", DefaultValueHandling = DefaultValueHandling.Ignore)] + public ResponseStatus Status { get; } + + /// + /// The unique ID of the function tool call generated by the model. + /// + [Preserve] + [JsonProperty("call_id", DefaultValueHandling = DefaultValueHandling.Ignore)] + public string CallId { get; } + + /// + /// A JSON string of the output of the function tool call. + /// + [Preserve] + [JsonProperty("output", DefaultValueHandling = DefaultValueHandling.Ignore)] + public string Output { get; } + + [Preserve] + public override string ToString() + => Output; + } +} From 017182b8628573c018d665c7fa342cb7f548bf76 Mon Sep 17 00:00:00 2001 From: Stephen Hodgson Date: Tue, 30 Sep 2025 12:04:20 -0400 Subject: [PATCH 14/22] update icons --- .../Runtime/Responses/CustomToolCall.cs.meta | 11 +++++++++++ .../Runtime/Responses/CustomToolCallOutput.cs.meta | 11 +++++++++++ .../Runtime/Responses/MCPToolRequireApproval.cs.meta | 11 +++++++++++ 3 files changed, 33 insertions(+) create mode 100644 OpenAI/Packages/com.openai.unity/Runtime/Responses/CustomToolCall.cs.meta create mode 100644 OpenAI/Packages/com.openai.unity/Runtime/Responses/CustomToolCallOutput.cs.meta create mode 100644 OpenAI/Packages/com.openai.unity/Runtime/Responses/MCPToolRequireApproval.cs.meta diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Responses/CustomToolCall.cs.meta b/OpenAI/Packages/com.openai.unity/Runtime/Responses/CustomToolCall.cs.meta new file mode 100644 index 00000000..698ee9c7 --- /dev/null +++ b/OpenAI/Packages/com.openai.unity/Runtime/Responses/CustomToolCall.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 638f39417c2d0204b8c25cbdce9e334e +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {fileID: 2800000, guid: 84a7eb8fc6eba7540bf56cea8e12249c, type: 3} + userData: + assetBundleName: + assetBundleVariant: diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Responses/CustomToolCallOutput.cs.meta b/OpenAI/Packages/com.openai.unity/Runtime/Responses/CustomToolCallOutput.cs.meta new file mode 100644 index 00000000..98a4e550 --- /dev/null +++ b/OpenAI/Packages/com.openai.unity/Runtime/Responses/CustomToolCallOutput.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 0001e30236c3dd84488872c942ed3cd8 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {fileID: 2800000, guid: 84a7eb8fc6eba7540bf56cea8e12249c, type: 3} + userData: + assetBundleName: + assetBundleVariant: diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Responses/MCPToolRequireApproval.cs.meta b/OpenAI/Packages/com.openai.unity/Runtime/Responses/MCPToolRequireApproval.cs.meta new file mode 100644 index 00000000..d05e2fd1 --- /dev/null +++ b/OpenAI/Packages/com.openai.unity/Runtime/Responses/MCPToolRequireApproval.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 9ec1d07cea8cdae44b76ce754d37c6b1 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {fileID: 2800000, guid: 84a7eb8fc6eba7540bf56cea8e12249c, type: 3} + userData: + assetBundleName: + assetBundleVariant: From f4b4f0b0da51851ef40407c798490cb5100ea472 Mon Sep 17 00:00:00 2001 From: Stephen Hodgson Date: Tue, 30 Sep 2025 23:43:30 -0400 Subject: [PATCH 15/22] refactor the responses behaviour with new conversation endpoint fixed some issues with image generation using responses image generation tool updated default model list make Model class Serializable so models can be picked and selected from inspectors refactor assistant/threads beta headers to only be applied on those specific endpoints --- .github/workflows/upm-subtree-split.yml | 2 + .../Runtime/Assistants/AssistantsEndpoint.cs | 26 +++- .../com.openai.unity/Runtime/Models/Model.cs | 140 +++++++++++++++++- .../com.openai.unity/Runtime/OpenAIClient.cs | 1 - .../Responses/CreateResponseRequest.cs | 58 ++++++++ .../Runtime/Responses/ImageGenerationCall.cs | 38 ++++- .../Runtime/Responses/ImageGenerationTool.cs | 18 +-- .../Runtime/Responses/Response.cs | 19 +++ .../Runtime/Responses/ResponseStatus.cs | 4 +- .../Runtime/Responses/ResponsesEndpoint.cs | 66 ++++++++- .../Runtime/Threads/ThreadsEndpoint.cs | 51 ++++--- .../Samples~/Responses/ResponsesBehaviour.cs | 86 +++++------ .../Tests/TestFixture_00_01_Authentication.cs | 2 +- 13 files changed, 410 insertions(+), 101 deletions(-) diff --git a/.github/workflows/upm-subtree-split.yml b/.github/workflows/upm-subtree-split.yml index f0d0775d..cfd5f359 100644 --- a/.github/workflows/upm-subtree-split.yml +++ b/.github/workflows/upm-subtree-split.yml @@ -6,6 +6,8 @@ on: jobs: upm-subtree-split: runs-on: ubuntu-latest + permissions: + contents: write steps: - uses: actions/checkout@v4 with: diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Assistants/AssistantsEndpoint.cs b/OpenAI/Packages/com.openai.unity/Runtime/Assistants/AssistantsEndpoint.cs index 6aa0a1fb..1c4411fd 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Assistants/AssistantsEndpoint.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Assistants/AssistantsEndpoint.cs @@ -2,6 +2,7 @@ using Newtonsoft.Json; using OpenAI.Extensions; +using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; using Utilities.WebRequestRest; @@ -10,7 +11,20 @@ namespace OpenAI.Assistants { public sealed class AssistantsEndpoint : OpenAIBaseEndpoint { - internal AssistantsEndpoint(OpenAIClient client) : base(client) { } + internal AssistantsEndpoint(OpenAIClient client) : base(client) + { + var assistantHeaders = new Dictionary(); + + foreach (var (key, value) in client.DefaultRequestHeaders) + { + assistantHeaders[key] = value; + } + + assistantHeaders["OpenAI-Beta"] = "assistants=v2"; + headers = assistantHeaders; + } + + private readonly IReadOnlyDictionary headers; protected override string Root => "assistants"; @@ -22,7 +36,7 @@ internal AssistantsEndpoint(OpenAIClient client) : base(client) { } /// public async Task> ListAssistantsAsync(ListQuery query = null, CancellationToken cancellationToken = default) { - var response = await Rest.GetAsync(GetUrl(queryParameters: query), parameters: new RestParameters(client.DefaultRequestHeaders), cancellationToken); + var response = await Rest.GetAsync(GetUrl(queryParameters: query), parameters: new RestParameters(headers), cancellationToken); response.Validate(EnableDebug); return response.Deserialize>(client); } @@ -58,7 +72,7 @@ public async Task CreateAssistantAsync(CreateAssistantRequest { request ??= new CreateAssistantRequest(); var payload = JsonConvert.SerializeObject(request, OpenAIClient.JsonSerializationOptions); - var response = await Rest.PostAsync(GetUrl(), payload, new RestParameters(client.DefaultRequestHeaders), cancellationToken); + var response = await Rest.PostAsync(GetUrl(), payload, new RestParameters(headers), cancellationToken); response.Validate(EnableDebug); return response.Deserialize(client); } @@ -71,7 +85,7 @@ public async Task CreateAssistantAsync(CreateAssistantRequest /// . public async Task RetrieveAssistantAsync(string assistantId, CancellationToken cancellationToken = default) { - var response = await Rest.GetAsync(GetUrl($"/{assistantId}"), new RestParameters(client.DefaultRequestHeaders), cancellationToken); + var response = await Rest.GetAsync(GetUrl($"/{assistantId}"), new RestParameters(headers), cancellationToken); response.Validate(EnableDebug); return response.Deserialize(client); } @@ -86,7 +100,7 @@ public async Task RetrieveAssistantAsync(string assistantId, public async Task ModifyAssistantAsync(string assistantId, CreateAssistantRequest request, CancellationToken cancellationToken = default) { var payload = JsonConvert.SerializeObject(request, OpenAIClient.JsonSerializationOptions); - var response = await Rest.PostAsync(GetUrl($"/{assistantId}"), payload, new RestParameters(client.DefaultRequestHeaders), cancellationToken); + var response = await Rest.PostAsync(GetUrl($"/{assistantId}"), payload, new RestParameters(headers), cancellationToken); response.Validate(EnableDebug); return response.Deserialize(client); } @@ -99,7 +113,7 @@ public async Task ModifyAssistantAsync(string assistantId, Cr /// True, if the assistant was deleted. public async Task DeleteAssistantAsync(string assistantId, CancellationToken cancellationToken = default) { - var response = await Rest.DeleteAsync(GetUrl($"/{assistantId}"), new RestParameters(client.DefaultRequestHeaders), cancellationToken); + var response = await Rest.DeleteAsync(GetUrl($"/{assistantId}"), new RestParameters(headers), cancellationToken); response.Validate(EnableDebug); var result = response.Deserialize(client); return result?.Deleted ?? false; diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Models/Model.cs b/OpenAI/Packages/com.openai.unity/Runtime/Models/Model.cs index 01239399..7878c198 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Models/Model.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Models/Model.cs @@ -3,6 +3,7 @@ using Newtonsoft.Json; using System; using System.Collections.Generic; +using UnityEngine; using UnityEngine.Scripting; namespace OpenAI.Models @@ -12,6 +13,7 @@ namespace OpenAI.Models /// /// [Preserve] + [Serializable] public sealed class Model { /// @@ -27,7 +29,7 @@ public Model(string id, string ownedBy = null) throw new ArgumentNullException(nameof(id), "Missing the id of the specified model."); } - Id = id; + this.id = id; OwnedBy = ownedBy; } @@ -38,11 +40,11 @@ internal Model( [JsonProperty("object")] string @object, [JsonProperty("created")] int createdAtUnixTimeSeconds, [JsonProperty("owned_by")] string ownedBy, - [JsonProperty("permission")] List permissions, - [JsonProperty("root")] string root, - [JsonProperty("parent")] string parent) - : this(id) + [JsonProperty("permission")] List permissions = null, + [JsonProperty("root")] string root = null, + [JsonProperty("parent")] string parent = null) { + this.id = id; Object = @object; OwnedBy = ownedBy; CreatedAtUnixTimeSeconds = createdAtUnixTimeSeconds; @@ -65,9 +67,12 @@ internal Model( /// public override string ToString() => Id; + [SerializeField] + private string id; + [Preserve] [JsonProperty("id")] - public string Id { get; } + public string Id => id; [Preserve] [JsonProperty("object")] @@ -145,6 +150,19 @@ internal Model( /// public static Model O3 { get; } = new("o3", "openai"); + /// + /// The o-series of models are trained with reinforcement learning to think before they answer and perform complex reasoning. + /// The o3-pro model uses more compute to think harder and provide consistently better answers.
+ /// o3-pro is available in the Responses API only to enable support for multi-turn model interactions before responding to API requests, + /// and other advanced API features in the future. Since o3-pro is designed to tackle tough problems, some requests may take several minutes to finish. + /// To avoid timeouts, try using background mode. + /// + /// + /// - Context Window: 200,000 tokens
+ /// - Max Output Tokens: 100,000 tokens + /// + public static Model O3Pro { get; } = new("o3-pro", "openai"); + /// /// o3-mini is our newest small reasoning model, providing high intelligence at the same cost and latency targets of o1-mini. /// o3-mini supports key developer features, like Structured Outputs, function calling, and Batch API. @@ -169,6 +187,15 @@ internal Model( #region Realtime Models + /// + /// This is our first general-availability realtime model, capable of responding to audio and text inputs in realtime over WebRTC, WebSocket, or SIP connections. + /// + /// + /// - Context Window: 32,000 tokens
+ /// - Max Output Tokens: 4,096 tokens + /// + public static Model GPT_Realtime { get; } = new("gpt-realtime", "openai"); + /// /// This is a preview release of the GPT-4o Realtime model, capable of responding to audio and text inputs in realtime over WebRTC or a WebSocket interface. /// @@ -191,6 +218,44 @@ internal Model( #region Chat Models + /// + /// GPT-5 is our flagship model for coding, reasoning, and agentic tasks across domains. + /// + /// + /// - Context Window: 400,000 context window
+ /// - Max Output Tokens: 128,000 max output tokens + /// + public static Model GPT_5 { get; } = new("gpt-5", "openai"); + + /// + /// GPT-5 mini is a faster, more cost-efficient version of GPT-5. It's great for well-defined tasks and precise prompts. + /// + /// + /// - Context Window: 400,000 context window
+ /// - Max Output Tokens: 128,000 max output tokens + /// + public static Model GPT_5_Mini { get; } = new("gpt-5-mini", "openai"); + + /// + /// GPT-5 Nano is our fastest, cheapest version of GPT-5. It's great for summarization and classification tasks. + /// + /// + /// - Context Window: 400,000 context window
+ /// - Max Output Tokens: 128,000 max output tokens + /// + public static Model GPT_5_Nano { get; } = new("gpt-5-nano", "openai"); + + /// + /// GPT-5 Chat points to the GPT-5 snapshot currently used in ChatGPT. + /// We recommend GPT-5 for most API usage, + /// but feel free to use this GPT-5 Chat model to test our latest improvements for chat use cases. + /// + /// + /// - Context Window: 128,000 context window
+ /// - Max Output Tokens: 16,384 max output tokens + /// + public static Model GPT_5_Chat { get; } = new("gpt-5-chat-latest", "openai"); + /// /// ChatGPT-4o points to the GPT-4o snapshot currently used in ChatGPT. /// GPT-4o is our versatile, high-intelligence flagship model. @@ -272,6 +337,14 @@ internal Model( /// public static Model GPT4_1_Nano { get; } = new("gpt-4.1-nano", "openai"); + /// + /// Deprecated - a research preview of GPT-4.5. We recommend using gpt-4.1 or o3 models instead for most use cases. + /// + /// + /// - Context Window: 128,000 context window
+ /// - Max Output Tokens: 16,384 max output tokens + /// + [Obsolete("Deprecated")] public static Model GPT4_5 { get; } = new("gpt-4.5-preview", "openai"); /// @@ -375,6 +448,16 @@ internal Model( #region Audio Models + /// + /// The gpt-audio model is our first generally available audio model. + /// It accepts audio inputs and outputs, and can be used in the Chat Completions REST API. + /// + /// + /// - Context Window: 128,000 tokens
+ /// - Max Output Tokens: 16,384 max output tokens + /// + public static Model GPT_Audio { get; } = new("gpt-audio", "openai"); + /// /// TTS is a model that converts text to natural sounding spoken text. /// The tts-1-hd model is optimized for high quality text-to-speech use cases. @@ -441,5 +524,50 @@ internal Model( public static Model DallE_2 { get; } = new("dall-e-2", "openai"); #endregion Image Models + + #region Specilized Models + + /// + /// GPT-5-Codex is a version of GPT-5 optimized for agentic coding tasks in Codex or similar environments. + /// It's available in the Responses API only and the underlying model snapshot will be regularly updated. + /// + /// + /// - Context Window: 400,000 tokens
+ /// - Max Output Tokens: 128,000 tokens + /// + public static Model GPT_5_Codex { get; } = new("gpt-5-codex", "openai"); + + /// + /// codex-mini-latest is a fine-tuned version of o4-mini specifically for use in Codex CLI. + /// + /// + /// - Context Window: 200,000 tokens
+ /// - Max Output Tokens: 100,000 tokens + /// + public static Model Codex_Mini_Latest { get; } = new("codex-mini-latest", "openai"); + + #endregion Specilized Models + + #region Open Weight Models + + /// + /// gpt-oss-120bis our most powerful open-weight model, which fits into a single H100 GPU (117B parameters with 5.1B active parameters). + /// + /// + /// - Context Window: 131,072 context window
+ /// - Max Output Tokens: 131,072 max output tokens + /// + public static Model GPT_OSS_120B { get; } = new("gpt-oss-120b", "openai"); + + /// + /// gpt-oss-20b is our medium-sized open-weight model for low latency, local, or specialized use-cases (21B parameters with 3.6B active parameters). + /// + /// + /// - Context Window: 131,072 context window
+ /// - Max Output Tokens: 131,072 max output tokens + /// + public static Model GPT_OSS_20B { get; } = new("gpt-oss-20b", "openai"); + + #endregion Open Weight Models } } diff --git a/OpenAI/Packages/com.openai.unity/Runtime/OpenAIClient.cs b/OpenAI/Packages/com.openai.unity/Runtime/OpenAIClient.cs index 542c23ab..23be99ee 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/OpenAIClient.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/OpenAIClient.cs @@ -73,7 +73,6 @@ protected override void SetupDefaultRequestHeaders() #if !UNITY_WEBGL { "User-Agent", "com.openai.unity" }, #endif - { "OpenAI-Beta", "assistants=v2" } }; if (Settings.Info.BaseRequestUrlFormat.Contains(OpenAISettingsInfo.OpenAIDomain) && diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Responses/CreateResponseRequest.cs b/OpenAI/Packages/com.openai.unity/Runtime/Responses/CreateResponseRequest.cs index a5e767f4..977b6893 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Responses/CreateResponseRequest.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Responses/CreateResponseRequest.cs @@ -75,6 +75,64 @@ public CreateResponseRequest( { } + [Preserve] + public CreateResponseRequest( + IResponseItem input, + Model model = null, + bool? background = null, + IEnumerable include = null, + string instructions = null, + int? maxOutputTokens = null, + IReadOnlyDictionary metadata = null, + bool? parallelToolCalls = null, + string previousResponseId = null, + Prompt prompt = null, + Reasoning reasoning = null, + string serviceTier = null, + bool? store = null, + double? temperature = null, + TextResponseFormat responseFormat = TextResponseFormat.Auto, + JsonSchema jsonSchema = null, + string toolChoice = null, + IEnumerable tools = null, + double? topP = null, + Truncation truncation = Truncation.Auto, + string user = null, + string conversationId = null, + int? maxToolCalls = null, + string promptCacheKey = null, + string safetyIdentifier = null, + int? topLogProbs = null) + : this( + input: new[] { input }, + model: model, + background: background, + include: include, + instructions: instructions, + maxOutputTokens: maxOutputTokens, + metadata: metadata, + parallelToolCalls: parallelToolCalls, + previousResponseId: previousResponseId, + prompt: prompt, + reasoning: reasoning, + serviceTier: serviceTier, + store: store, + temperature: temperature, + responseFormat: responseFormat, + jsonSchema: jsonSchema, + toolChoice: toolChoice, + tools: tools, + topP: topP, + truncation: truncation, + user: user, + conversationId: conversationId, + maxToolCalls: maxToolCalls, + promptCacheKey: promptCacheKey, + safetyIdentifier: safetyIdentifier, + topLogProbs: topLogProbs) + { + } + [Preserve] public CreateResponseRequest( IEnumerable input, diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Responses/ImageGenerationCall.cs b/OpenAI/Packages/com.openai.unity/Runtime/Responses/ImageGenerationCall.cs index 8fc2edd6..fdac4732 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Responses/ImageGenerationCall.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Responses/ImageGenerationCall.cs @@ -19,13 +19,25 @@ internal ImageGenerationCall( [JsonProperty("type")] ResponseItemType type, [JsonProperty("object")] string @object, [JsonProperty("status")] ResponseStatus status, - [JsonProperty("result")] string result) + [JsonProperty("result")] string result, + [JsonProperty("partial_image_b64")] string partialImageResult = null, + [JsonProperty("output_format")] string outputFormat = null, + [JsonProperty("revised_prompt")] string revisedPrompt = null, + [JsonProperty("background")] string background = null, + [JsonProperty("size")] string size = null, + [JsonProperty("quality")] string quality = null) { Id = id; Type = type; Object = @object; Status = status; Result = result; + PartialImageResult = partialImageResult; + OutputFormat = outputFormat; + RevisedPrompt = revisedPrompt; + Background = background; + Size = size; + Quality = quality; } /// @@ -55,6 +67,30 @@ internal ImageGenerationCall( [JsonProperty("result")] public string Result { get; } + [Preserve] + [JsonProperty("partial_image_b64")] + public string PartialImageResult { get; } + + [Preserve] + [JsonProperty("output_format")] + public string OutputFormat { get; } + + [Preserve] + [JsonProperty("revised_prompt")] + public string RevisedPrompt { get; } + + [Preserve] + [JsonProperty("background")] + public string Background { get; } + + [Preserve] + [JsonProperty("size")] + public string Size { get; } + + [Preserve] + [JsonProperty("quality")] + public string Quality { get; } + [Preserve] [JsonIgnore] public Texture2D Image { get; internal set; } diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Responses/ImageGenerationTool.cs b/OpenAI/Packages/com.openai.unity/Runtime/Responses/ImageGenerationTool.cs index 57aa26a0..74b24922 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Responses/ImageGenerationTool.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Responses/ImageGenerationTool.cs @@ -42,15 +42,15 @@ public ImageGenerationTool( [JsonConstructor] internal ImageGenerationTool( [JsonProperty("type")] string type, - [JsonProperty("background", DefaultValueHandling = DefaultValueHandling.Ignore)] string background, - [JsonProperty("input_image_mask", DefaultValueHandling = DefaultValueHandling.Ignore)] InputImageMask inputImageMask, - [JsonProperty("model", DefaultValueHandling = DefaultValueHandling.Ignore)] string model, - [JsonProperty("moderation", DefaultValueHandling = DefaultValueHandling.Ignore)] string moderation, - [JsonProperty("output_compression", DefaultValueHandling = DefaultValueHandling.Ignore)] int? outputCompression, - [JsonProperty("output_format", DefaultValueHandling = DefaultValueHandling.Ignore)] string outputFormat, - [JsonProperty("partial_images", DefaultValueHandling = DefaultValueHandling.Ignore)] int? partialImages, - [JsonProperty("quality", DefaultValueHandling = DefaultValueHandling.Ignore)] string quality, - [JsonProperty("size", DefaultValueHandling = DefaultValueHandling.Ignore)] string size) + [JsonProperty("background")] string background, + [JsonProperty("input_image_mask")] InputImageMask inputImageMask, + [JsonProperty("model")] string model, + [JsonProperty("moderation")] string moderation, + [JsonProperty("output_compression")] int? outputCompression, + [JsonProperty("output_format")] string outputFormat, + [JsonProperty("partial_images")] int? partialImages, + [JsonProperty("quality")] string quality, + [JsonProperty("size")] string size) { Type = type; Background = background; diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Responses/Response.cs b/OpenAI/Packages/com.openai.unity/Runtime/Responses/Response.cs index 5da85943..c71be352 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Responses/Response.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Responses/Response.cs @@ -365,6 +365,25 @@ internal void InsertOutputItem(IResponseItem item, int index) output.Insert(index, item); } + [Preserve] + internal void UpdateOutputItem(IResponseItem item, int index) + { + if (item == null) + { + throw new ArgumentNullException(nameof(item)); + } + + if (index >= output.Count) + { + for (var i = output.Count; i <= index; i++) + { + output.Add(null); + } + } + + output[index] = item; + } + [Preserve] public void PrintUsage() { diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Responses/ResponseStatus.cs b/OpenAI/Packages/com.openai.unity/Runtime/Responses/ResponseStatus.cs index 1262f4bb..f1bcf858 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Responses/ResponseStatus.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Responses/ResponseStatus.cs @@ -17,11 +17,13 @@ public enum ResponseStatus InProgress, [EnumMember(Value = "searching")] Searching, + [EnumMember(Value = "generating")] + Generating, [EnumMember(Value = "cancelled")] Cancelled, [EnumMember(Value = "queued")] Queued, [EnumMember(Value = "incomplete")] - Incomplete + Incomplete, } } diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Responses/ResponsesEndpoint.cs b/OpenAI/Packages/com.openai.unity/Runtime/Responses/ResponsesEndpoint.cs index 6553d1d6..e9be2627 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Responses/ResponsesEndpoint.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Responses/ResponsesEndpoint.cs @@ -158,6 +158,7 @@ private async Task StreamResponseAsync(string endpoint, string payload case "response.completed": case "response.failed": case "response.incomplete": + { var partialResponse = sseResponse.Deserialize(@object["response"], client); if (response == null) @@ -178,8 +179,10 @@ private async Task StreamResponseAsync(string endpoint, string payload serverSentEvent = response; break; + } case "response.content_part.added": case "response.content_part.done": + { var part = sseResponse.Deserialize(@object["part"], client); var messageItem = (Message)response!.Output[outputIndex!.Value]; @@ -195,18 +198,41 @@ private async Task StreamResponseAsync(string endpoint, string payload serverSentEvent = part; } break; + } case "response.output_item.added": case "response.output_item.done": + { var item = sseResponse.Deserialize(@object["item"], client); - response!.InsertOutputItem(item, outputIndex!.Value); if (@event == "response.output_item.done") { - serverSentEvent = item; + if (item.Type == ResponseItemType.ImageGenerationCall && + item is ImageGenerationCall imageGenerationCall) + { + if (!string.IsNullOrWhiteSpace(imageGenerationCall.Result)) + { + var (texture, _) = await TextureExtensions.ConvertFromBase64Async(imageGenerationCall.Result, EnableDebug, cancellationToken); + imageGenerationCall.Image = texture; + } + + response!.UpdateOutputItem(imageGenerationCall, outputIndex!.Value); + serverSentEvent = imageGenerationCall; + } + else + { + response!.UpdateOutputItem(item, outputIndex!.Value); + serverSentEvent = item; + } + } + else // response.output_item.added + { + response!.InsertOutputItem(item, outputIndex!.Value); } break; + } case "response.function_call_arguments.delta": case "response.function_call_arguments.done": + { var functionToolCall = (FunctionToolCall)response!.Output[outputIndex!.Value]; if (functionToolCall.Id != itemId) @@ -219,6 +245,27 @@ private async Task StreamResponseAsync(string endpoint, string payload functionToolCall.Delta = delta; } break; + } + case "response.image_generation_call.in_progress": + case "response.image_generation_call.generating": + case "response.image_generation_call.partial_image": + { + var imageGenerationCall = (ImageGenerationCall)response!.Output[outputIndex!.Value]; + + if (imageGenerationCall.Id != itemId) + { + throw new InvalidOperationException($"ImageGenerationCall ID mismatch! Expected: {imageGenerationCall.Id}, got: {itemId}"); + } + + //if (!string.IsNullOrWhiteSpace(imageGenerationCall.PartialImageResult)) + //{ + // var (texture, _) = await TextureExtensions.ConvertFromBase64Async(imageGenerationCall.PartialImageResult, EnableDebug, cancellationToken); + // imageGenerationCall.Image = texture; + //} + + serverSentEvent = imageGenerationCall; + break; + } case "response.audio.delta": case "response.audio.done": case "response.audio.transcript.delta": @@ -230,7 +277,8 @@ private async Task StreamResponseAsync(string endpoint, string payload case "response.refusal.done": case "response.reasoning_summary_text.delta": case "response.reasoning_summary_text.done": - messageItem = (Message)response!.Output[outputIndex!.Value]; + { + var messageItem = (Message)response!.Output[outputIndex!.Value]; if (messageItem.Id != itemId) { @@ -243,25 +291,30 @@ private async Task StreamResponseAsync(string endpoint, string payload { case AudioContent audioContent: AudioContent partialContent; + switch (@event) { case "response.audio.delta": partialContent = new AudioContent(audioContent.Type, base64Data: delta); audioContent.AppendFrom(partialContent); serverSentEvent = partialContent; + break; case "response.audio.transcript.delta": partialContent = new AudioContent(audioContent.Type, transcript: delta); audioContent.AppendFrom(partialContent); serverSentEvent = partialContent; + break; case "response.audio.done": case "response.audio.transcript.done": serverSentEvent = audioContent; + break; default: throw new InvalidOperationException($"Unexpected event type: {@event} for AudioContent."); } + break; case TextContent textContent: if (!string.IsNullOrWhiteSpace(text)) @@ -280,6 +333,7 @@ private async Task StreamResponseAsync(string endpoint, string payload } serverSentEvent = textContent; + break; case RefusalContent refusalContent: var refusal = @object["refusal"]?.Value(); @@ -295,6 +349,7 @@ private async Task StreamResponseAsync(string endpoint, string payload } serverSentEvent = refusalContent; + break; case ReasoningContent reasoningContent: if (!string.IsNullOrWhiteSpace(text)) @@ -309,7 +364,9 @@ private async Task StreamResponseAsync(string endpoint, string payload break; } + break; + } case "response.reasoning_summary_part.added": case "response.reasoning_summary_part.done": var summaryIndex = @object["summary_index"]!.Value(); @@ -351,9 +408,6 @@ private async Task StreamResponseAsync(string endpoint, string payload case "response.file_search_call.in_progress": case "response.file_search_call.searching": case "response.file_search_call.completed": - case "response.image_generation_call.in_progress": - case "response.image_generation_call.generating": - case "response.image_generation_call.partial_image": case "response.image_generation_call.completed": case "response.mcp_call_arguments.delta": case "response.mcp_call_arguments.done": diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Threads/ThreadsEndpoint.cs b/OpenAI/Packages/com.openai.unity/Runtime/Threads/ThreadsEndpoint.cs index eebab1e7..1e8ce6ea 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Threads/ThreadsEndpoint.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Threads/ThreadsEndpoint.cs @@ -19,7 +19,20 @@ namespace OpenAI.Threads /// public sealed class ThreadsEndpoint : OpenAIBaseEndpoint { - internal ThreadsEndpoint(OpenAIClient client) : base(client) { } + internal ThreadsEndpoint(OpenAIClient client) : base(client) + { + var assistantHeaders = new Dictionary(); + + foreach (var (key, value) in client.DefaultRequestHeaders) + { + assistantHeaders[key] = value; + } + + assistantHeaders["OpenAI-Beta"] = "assistants=v2"; + headers = assistantHeaders; + } + + private readonly IReadOnlyDictionary headers; protected override string Root => "threads"; @@ -32,7 +45,7 @@ internal ThreadsEndpoint(OpenAIClient client) : base(client) { } public async Task CreateThreadAsync(CreateThreadRequest request = null, CancellationToken cancellationToken = default) { var payload = request != null ? JsonConvert.SerializeObject(request, OpenAIClient.JsonSerializationOptions) : string.Empty; - var response = await Rest.PostAsync(GetUrl(), payload, new RestParameters(client.DefaultRequestHeaders), cancellationToken); + var response = await Rest.PostAsync(GetUrl(), payload, new RestParameters(headers), cancellationToken); response.Validate(EnableDebug); return response.Deserialize(client); } @@ -45,7 +58,7 @@ public async Task CreateThreadAsync(CreateThreadRequest request /// . public async Task RetrieveThreadAsync(string threadId, CancellationToken cancellationToken = default) { - var response = await Rest.GetAsync(GetUrl($"/{threadId}"), new RestParameters(client.DefaultRequestHeaders), cancellationToken); + var response = await Rest.GetAsync(GetUrl($"/{threadId}"), new RestParameters(headers), cancellationToken); response.Validate(EnableDebug); return response.Deserialize(client); } @@ -66,7 +79,7 @@ public async Task RetrieveThreadAsync(string threadId, Cancellat public async Task ModifyThreadAsync(string threadId, IReadOnlyDictionary metadata, CancellationToken cancellationToken = default) { var payload = JsonConvert.SerializeObject(new { metadata }, OpenAIClient.JsonSerializationOptions); - var response = await Rest.PostAsync(GetUrl($"/{threadId}"), payload, new RestParameters(client.DefaultRequestHeaders), cancellationToken); + var response = await Rest.PostAsync(GetUrl($"/{threadId}"), payload, new RestParameters(headers), cancellationToken); response.Validate(EnableDebug); return response.Deserialize(client); } @@ -79,7 +92,7 @@ public async Task ModifyThreadAsync(string threadId, IReadOnlyDi /// True, if was successfully deleted. public async Task DeleteThreadAsync(string threadId, CancellationToken cancellationToken = default) { - var response = await Rest.DeleteAsync(GetUrl($"/{threadId}"), new RestParameters(client.DefaultRequestHeaders), cancellationToken); + var response = await Rest.DeleteAsync(GetUrl($"/{threadId}"), new RestParameters(headers), cancellationToken); response.Validate(EnableDebug); return response.Deserialize(client)?.Deleted ?? false; } @@ -96,7 +109,7 @@ public async Task DeleteThreadAsync(string threadId, CancellationToken can public async Task CreateMessageAsync(string threadId, Message message, CancellationToken cancellationToken = default) { var payload = JsonConvert.SerializeObject(message, OpenAIClient.JsonSerializationOptions); - var response = await Rest.PostAsync(GetUrl($"/{threadId}/messages"), payload, new RestParameters(client.DefaultRequestHeaders), cancellationToken); + var response = await Rest.PostAsync(GetUrl($"/{threadId}/messages"), payload, new RestParameters(headers), cancellationToken); response.Validate(EnableDebug); return response.Deserialize(client); } @@ -119,7 +132,7 @@ public async Task> ListMessagesAsync(string thread queryParams.Add("run_id", runId); } - var response = await Rest.GetAsync(GetUrl($"/{threadId}/messages", queryParams), new RestParameters(client.DefaultRequestHeaders), cancellationToken); + var response = await Rest.GetAsync(GetUrl($"/{threadId}/messages", queryParams), new RestParameters(headers), cancellationToken); response.Validate(EnableDebug); return response.Deserialize>(client); } @@ -133,7 +146,7 @@ public async Task> ListMessagesAsync(string thread /// . public async Task RetrieveMessageAsync(string threadId, string messageId, CancellationToken cancellationToken = default) { - var response = await Rest.GetAsync(GetUrl($"/{threadId}/messages/{messageId}"), new RestParameters(client.DefaultRequestHeaders), cancellationToken); + var response = await Rest.GetAsync(GetUrl($"/{threadId}/messages/{messageId}"), new RestParameters(headers), cancellationToken); response.Validate(EnableDebug); return response.Deserialize(client); } @@ -171,7 +184,7 @@ public Task ModifyMessageAsync(MessageResponse message, IReadOn public async Task ModifyMessageAsync(string threadId, string messageId, IReadOnlyDictionary metadata, CancellationToken cancellationToken = default) { var payload = JsonConvert.SerializeObject(new { metadata }, OpenAIClient.JsonSerializationOptions); - var response = await Rest.PostAsync(GetUrl($"/{threadId}/messages/{messageId}"), payload, new RestParameters(client.DefaultRequestHeaders), cancellationToken); + var response = await Rest.PostAsync(GetUrl($"/{threadId}/messages/{messageId}"), payload, new RestParameters(headers), cancellationToken); response.Validate(EnableDebug); return response.Deserialize(client); } @@ -189,7 +202,7 @@ public async Task ModifyMessageAsync(string threadId, string me /// public async Task> ListRunsAsync(string threadId, ListQuery query = null, CancellationToken cancellationToken = default) { - var response = await Rest.GetAsync(GetUrl($"/{threadId}/runs", query), new RestParameters(client.DefaultRequestHeaders), cancellationToken); + var response = await Rest.GetAsync(GetUrl($"/{threadId}/runs", query), new RestParameters(headers), cancellationToken); response.Validate(EnableDebug); return response.Deserialize>(client); } @@ -268,7 +281,7 @@ public async Task CreateRunAsync(string threadId, CreateRunRequest return await StreamRunAsync(endpoint, payload, streamEventHandler, cancellationToken); } - var response = await Rest.PostAsync(endpoint, payload, new RestParameters(client.DefaultRequestHeaders), cancellationToken); + var response = await Rest.PostAsync(endpoint, payload, new RestParameters(headers), cancellationToken); response.Validate(EnableDebug); return response.Deserialize(client); } @@ -343,7 +356,7 @@ public async Task CreateThreadAndRunAsync(CreateThreadAndRunRequest return await StreamRunAsync(endpoint, payload, streamEventHandler, cancellationToken); } - var response = await Rest.PostAsync(endpoint, payload, new RestParameters(client.DefaultRequestHeaders), cancellationToken); + var response = await Rest.PostAsync(endpoint, payload, new RestParameters(headers), cancellationToken); response.Validate(EnableDebug); return response.Deserialize(client); } @@ -357,7 +370,7 @@ public async Task CreateThreadAndRunAsync(CreateThreadAndRunRequest /// . public async Task RetrieveRunAsync(string threadId, string runId, CancellationToken cancellationToken = default) { - var response = await Rest.GetAsync(GetUrl($"/{threadId}/runs/{runId}"), new RestParameters(client.DefaultRequestHeaders), cancellationToken); + var response = await Rest.GetAsync(GetUrl($"/{threadId}/runs/{runId}"), new RestParameters(headers), cancellationToken); response.Validate(EnableDebug); return response.Deserialize(client); } @@ -378,7 +391,7 @@ public async Task RetrieveRunAsync(string threadId, string runId, C public async Task ModifyRunAsync(string threadId, string runId, IReadOnlyDictionary metadata, CancellationToken cancellationToken = default) { var payload = JsonConvert.SerializeObject(new { metadata }, OpenAIClient.JsonSerializationOptions); - var response = await Rest.PostAsync(GetUrl($"/{threadId}/runs/{runId}"), payload, new RestParameters(client.DefaultRequestHeaders), cancellationToken); + var response = await Rest.PostAsync(GetUrl($"/{threadId}/runs/{runId}"), payload, new RestParameters(headers), cancellationToken); response.Validate(EnableDebug); return response.Deserialize(client); } @@ -419,7 +432,7 @@ public async Task SubmitToolOutputsAsync(string threadId, string ru return await StreamRunAsync(endpoint, payload, streamEventHandler, cancellationToken); } - var response = await Rest.PostAsync(endpoint, payload, new RestParameters(client.DefaultRequestHeaders), cancellationToken); + var response = await Rest.PostAsync(endpoint, payload, new RestParameters(headers), cancellationToken); response.Validate(EnableDebug); return response.Deserialize(client); } @@ -434,7 +447,7 @@ public async Task SubmitToolOutputsAsync(string threadId, string ru /// . public async Task> ListRunStepsAsync(string threadId, string runId, ListQuery query = null, CancellationToken cancellationToken = default) { - var response = await Rest.GetAsync(GetUrl($"/{threadId}/runs/{runId}/steps", query), new RestParameters(client.DefaultRequestHeaders), cancellationToken); + var response = await Rest.GetAsync(GetUrl($"/{threadId}/runs/{runId}/steps", query), new RestParameters(headers), cancellationToken); response.Validate(EnableDebug); return response.Deserialize>(client); } @@ -449,7 +462,7 @@ public async Task> ListRunStepsAsync(string thread /// . public async Task RetrieveRunStepAsync(string threadId, string runId, string stepId, CancellationToken cancellationToken = default) { - var response = await Rest.GetAsync(GetUrl($"/{threadId}/runs/{runId}/steps/{stepId}"), new RestParameters(client.DefaultRequestHeaders), cancellationToken); + var response = await Rest.GetAsync(GetUrl($"/{threadId}/runs/{runId}/steps/{stepId}"), new RestParameters(headers), cancellationToken); response.Validate(EnableDebug); return response.Deserialize(client); } @@ -463,7 +476,7 @@ public async Task RetrieveRunStepAsync(string threadId, string /// . public async Task CancelRunAsync(string threadId, string runId, CancellationToken cancellationToken = default) { - var response = await Rest.PostAsync(GetUrl($"/{threadId}/runs/{runId}/cancel"), string.Empty, new RestParameters(client.DefaultRequestHeaders), cancellationToken); + var response = await Rest.PostAsync(GetUrl($"/{threadId}/runs/{runId}/cancel"), string.Empty, new RestParameters(headers), cancellationToken); response.Validate(EnableDebug); var run = response.Deserialize(client); @@ -590,7 +603,7 @@ private async Task StreamRunAsync(string endpoint, string payload, await streamEventHandler.Invoke(@event, serverSentEvent); } // ReSharper restore AccessToModifiedClosure - }, new RestParameters(client.DefaultRequestHeaders), cancellationToken); + }, new RestParameters(headers), cancellationToken); response.Validate(EnableDebug); if (run == null) { return null; } run = await run.WaitForStatusChangeAsync(timeout: -1, cancellationToken: cancellationToken); diff --git a/OpenAI/Packages/com.openai.unity/Samples~/Responses/ResponsesBehaviour.cs b/OpenAI/Packages/com.openai.unity/Samples~/Responses/ResponsesBehaviour.cs index 303a8524..f6cc5ea3 100644 --- a/OpenAI/Packages/com.openai.unity/Samples~/Responses/ResponsesBehaviour.cs +++ b/OpenAI/Packages/com.openai.unity/Samples~/Responses/ResponsesBehaviour.cs @@ -1,8 +1,6 @@ // Licensed under the MIT License. See LICENSE in the project root for license information. -using Newtonsoft.Json; using OpenAI.Audio; -using OpenAI.Images; using OpenAI.Models; using OpenAI.Responses; using System; @@ -56,12 +54,11 @@ public class ResponsesBehaviour : MonoBehaviour [SerializeField] [TextArea(3, 10)] - private string systemPrompt = "You are a helpful assistant.\n- If an image is requested then use \"![Image](output.jpg)\" to display it.\n- When performing function calls, use the defaults unless explicitly told to use a specific value.\n- Images should always be generated in base64."; + private string systemPrompt = "You are a helpful assistant.\n- If an image is requested then use \"![Image](output.jpg)\" to display it.\n- When performing tool calls, tell the user what you're doing before calling, and always use the defaults unless explicitly told to use a specific value."; private OpenAIClient openAI; - private readonly List conversation = new(); + private Conversation conversation; - private Tool imageTool; private readonly List assistantTools = new(); #if !UNITY_2022_3_OR_NEWER @@ -84,20 +81,31 @@ private void OnValidate() } } - private void Awake() + private async void Awake() { OnValidate(); + openAI = new OpenAIClient(configuration) { EnableDebug = enableDebug }; + RecordingManager.EnableDebug = enableDebug; - imageTool = Tool.GetOrCreateTool(openAI.ImagesEndPoint, nameof(ImagesEndpoint.GenerateImageAsync)); - assistantTools.Add(imageTool); - conversation.Add(new Message(Role.Developer, systemPrompt)); + assistantTools.Add(new ImageGenerationTool()); inputField.onSubmit.AddListener(SubmitChat); submitButton.onClick.AddListener(SubmitChat); recordButton.onClick.AddListener(ToggleRecording); + + try + { + conversation = await openAI.ConversationsEndpoint.CreateConversationAsync( + new CreateConversationRequest(new Message(Role.Developer, systemPrompt)), + destroyCancellationToken); + } + catch (Exception e) + { + Debug.LogException(e); + } } #if !UNITY_2022_3_OR_NEWER @@ -119,34 +127,31 @@ private async void SubmitChat() inputField.ReleaseSelection(); inputField.interactable = false; submitButton.interactable = false; - conversation.Add(new Message(Role.User, inputField.text)); + var userInput = inputField.text; var userMessageContent = AddNewTextMessageContent(Role.User); - userMessageContent.text = $"User: {inputField.text}"; + userMessageContent.text = $"User: {userInput}"; inputField.text = string.Empty; try { - var request = new CreateResponseRequest(input: conversation, tools: assistantTools, model: Model.GPT4_1_Nano); + var request = new CreateResponseRequest(textInput: userInput, conversationId: conversation, tools: assistantTools, model: Model.GPT4_1_Nano); async Task StreamEventHandler(string eventName, IServerSentEvent sseEvent) { switch (sseEvent) { case Message messageItem: - conversation.Add(messageItem); var assistantMessageContent = AddNewTextMessageContent(Role.Assistant); var message = messageItem.ToString().Replace("![Image](output.jpg)", string.Empty); assistantMessageContent.text = $"Assistant: {message}"; scrollView.verticalNormalizedPosition = 0f; await GenerateSpeechAsync(message, destroyCancellationToken); break; - case FunctionToolCall functionToolCall: - conversation.Add(functionToolCall); - var output = await ProcessToolCallAsync(functionToolCall, destroyCancellationToken); - conversation.Add(output); - await openAI.ResponsesEndpoint.CreateModelResponseAsync( - request: new(input: conversation, tools: assistantTools, toolChoice: "none", model: Model.GPT4_1_Nano), - streamEventHandler: StreamEventHandler, - cancellationToken: destroyCancellationToken); + case ImageGenerationCall imageGenerationCall: + if (imageGenerationCall.Image != null) + { + AddNewImageContent(imageGenerationCall.Image); + scrollView.verticalNormalizedPosition = 0f; + } break; } } @@ -178,44 +183,18 @@ await openAI.ResponsesEndpoint.CreateModelResponseAsync( } } - private async Task ProcessToolCallAsync(FunctionToolCall toolCall, CancellationToken cancellationToken) + private async Task GenerateSpeechAsync(string text, CancellationToken cancellationToken) { try { - if (toolCall.Name == imageTool.Function.Name) + if (string.IsNullOrWhiteSpace(text)) { - var output = await toolCall.InvokeFunctionAsync>(cancellationToken: cancellationToken); - - foreach (var imageResult in output.OutputResult) - { - AddNewImageContent(imageResult); - } - - scrollView.verticalNormalizedPosition = 0f; - return output; + return; } - return await toolCall.InvokeFunctionAsync(cancellationToken); - } - catch (Exception e) - { - Debug.LogError(e); - return new FunctionToolCallOutput(toolCall.CallId, JsonConvert.SerializeObject(new { error = new Error(e) })); - } - } - - private async Task GenerateSpeechAsync(string text, CancellationToken cancellationToken) - { - try - { - if (string.IsNullOrWhiteSpace(text)) { return; } - var request = new SpeechRequest(input: text, model: Model.TTS_1, voice: voice, responseFormat: SpeechResponseFormat.PCM); var stopwatch = Stopwatch.StartNew(); - var speechClip = await openAI.AudioEndpoint.GetSpeechAsync(request, partialClip => - { - streamAudioSource.BufferCallback(partialClip.AudioSamples); - }, cancellationToken); + var speechClip = await openAI.AudioEndpoint.GetSpeechAsync(request, partialClip => { streamAudioSource.BufferCallback(partialClip.AudioSamples); }, cancellationToken); var playbackTime = speechClip.Length - (float)stopwatch.Elapsed.TotalSeconds + 0.1f; await Awaiters.DelayAsync(TimeSpan.FromSeconds(playbackTime), cancellationToken).ConfigureAwait(true); @@ -235,9 +214,14 @@ private async Task GenerateSpeechAsync(string text, CancellationToken cancellati break; default: Debug.LogError(e); + break; } } + finally + { + ((AudioSource)streamAudioSource).clip = null; + } } private TextMeshProUGUI AddNewTextMessageContent(Role role) diff --git a/OpenAI/Packages/com.openai.unity/Tests/TestFixture_00_01_Authentication.cs b/OpenAI/Packages/com.openai.unity/Tests/TestFixture_00_01_Authentication.cs index f7c10bdf..efed134f 100644 --- a/OpenAI/Packages/com.openai.unity/Tests/TestFixture_00_01_Authentication.cs +++ b/OpenAI/Packages/com.openai.unity/Tests/TestFixture_00_01_Authentication.cs @@ -206,7 +206,7 @@ public void Test_12_CustomDomainConfigurationSettings() Debug.Log(api.Settings.Info.BaseRequestUrlFormat); Debug.Log(api.Settings.Info.BaseWebSocketUrlFormat); Assert.AreEqual($"https://{domain}/v1/{{0}}", api.Settings.Info.BaseRequestUrlFormat); - Assert.AreEqual($"wss://{domain}/v1/{{0}}", api.Settings.Info.BaseWebSocketUrlFormat); + Assert.AreEqual($"wss://{OpenAISettingsInfo.OpenAIDomain}/v1/{{0}}", api.Settings.Info.BaseWebSocketUrlFormat); } [TearDown] From 184c926baf9e50e737afa79508a7ff7438a43562 Mon Sep 17 00:00:00 2001 From: StephenHodgson Date: Wed, 1 Oct 2025 16:46:19 -0400 Subject: [PATCH 16/22] update com.utilities.rest 4.1.0 --- OpenAI/Packages/com.openai.unity/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/OpenAI/Packages/com.openai.unity/package.json b/OpenAI/Packages/com.openai.unity/package.json index 9c8bc405..f89d24c3 100644 --- a/OpenAI/Packages/com.openai.unity/package.json +++ b/OpenAI/Packages/com.openai.unity/package.json @@ -19,7 +19,7 @@ "dependencies": { "com.utilities.audio": "2.2.8", "com.utilities.encoder.wav": "2.2.4", - "com.utilities.rest": "4.0.0", + "com.utilities.rest": "4.1.0", "com.utilities.websockets": "1.0.6" }, "samples": [ From a95e4220aab69127a04e06b89588f024392cb3c7 Mon Sep 17 00:00:00 2001 From: Stephen Hodgson Date: Thu, 2 Oct 2025 17:47:07 -0400 Subject: [PATCH 17/22] more refactoring for image gen tool --- .../Runtime/Extensions/TextureExtensions.cs | 8 +- .../Runtime/Responses/ImageGenerationCall.cs | 49 +++-- .../Runtime/Responses/ImageGenerationTool.cs | 13 ++ .../Runtime/Responses/Response.cs | 19 -- .../Runtime/Responses/ResponsesEndpoint.cs | 71 +++---- .../Samples~/Responses/ResponsesBehaviour.cs | 5 +- .../Tests/OpenAI.Tests.asmdef | 3 +- .../Tests/TestFixture_14_Responses.cs | 176 ++++++++++++------ 8 files changed, 204 insertions(+), 140 deletions(-) diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Extensions/TextureExtensions.cs b/OpenAI/Packages/com.openai.unity/Runtime/Extensions/TextureExtensions.cs index 8d90c32e..6ccbf2b8 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Extensions/TextureExtensions.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Extensions/TextureExtensions.cs @@ -11,22 +11,22 @@ namespace OpenAI.Extensions { internal static class TextureExtensions { - public static async Task<(Texture2D, string)> ConvertFromBase64Async(string data, bool debug, CancellationToken cancellationToken) + public static async Task<(Texture2D, string)> ConvertFromBase64Async(string b64, bool debug, CancellationToken cancellationToken) { - var imageData = Convert.FromBase64String(data); + var imageData = Convert.FromBase64String(b64); #if PLATFORM_WEBGL var texture = new Texture2D(2, 2); texture.LoadImage(imageData); return await Task.FromResult((texture, string.Empty)); #else - if (!Rest.TryGetDownloadCacheItem(data, out var localFilePath)) + if (!Rest.TryGetDownloadCacheItem(b64, out var localFilePath)) { await File.WriteAllBytesAsync(localFilePath, imageData, cancellationToken).ConfigureAwait(true); localFilePath = $"file://{localFilePath}"; } var texture = await Rest.DownloadTextureAsync(localFilePath, parameters: new RestParameters(debug: debug), cancellationToken: cancellationToken); - Rest.TryGetDownloadCacheItem(data, out var cachedPath); + Rest.TryGetDownloadCacheItem(b64, out var cachedPath); return (texture, cachedPath); #endif } diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Responses/ImageGenerationCall.cs b/OpenAI/Packages/com.openai.unity/Runtime/Responses/ImageGenerationCall.cs index fdac4732..13ae98ea 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Responses/ImageGenerationCall.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Responses/ImageGenerationCall.cs @@ -1,6 +1,9 @@ // Licensed under the MIT License. See LICENSE in the project root for license information. using Newtonsoft.Json; +using OpenAI.Extensions; +using System.Threading; +using System.Threading.Tasks; using UnityEngine; using UnityEngine.Scripting; @@ -20,6 +23,7 @@ internal ImageGenerationCall( [JsonProperty("object")] string @object, [JsonProperty("status")] ResponseStatus status, [JsonProperty("result")] string result, + [JsonProperty("partial_image_index")] int? partialImageIndex = null, [JsonProperty("partial_image_b64")] string partialImageResult = null, [JsonProperty("output_format")] string outputFormat = null, [JsonProperty("revised_prompt")] string revisedPrompt = null, @@ -64,35 +68,50 @@ internal ImageGenerationCall( /// The generated image encoded in base64. /// [Preserve] - [JsonProperty("result")] + [JsonProperty("result", DefaultValueHandling = DefaultValueHandling.Ignore)] public string Result { get; } [Preserve] - [JsonProperty("partial_image_b64")] - public string PartialImageResult { get; } + [JsonProperty("partial_image_index", DefaultValueHandling = DefaultValueHandling.Ignore)] + public int? PartialImageIndex { get; internal set; } [Preserve] - [JsonProperty("output_format")] - public string OutputFormat { get; } + [JsonProperty("partial_image_b64", DefaultValueHandling = DefaultValueHandling.Ignore)] + public string PartialImageResult { get; internal set; } [Preserve] - [JsonProperty("revised_prompt")] - public string RevisedPrompt { get; } + [JsonProperty("output_format", DefaultValueHandling = DefaultValueHandling.Ignore)] + public string OutputFormat { get; internal set; } [Preserve] - [JsonProperty("background")] - public string Background { get; } + [JsonProperty("revised_prompt", DefaultValueHandling = DefaultValueHandling.Ignore)] + public string RevisedPrompt { get; internal set; } [Preserve] - [JsonProperty("size")] - public string Size { get; } + [JsonProperty("background", DefaultValueHandling = DefaultValueHandling.Ignore)] + public string Background { get; internal set; } [Preserve] - [JsonProperty("quality")] - public string Quality { get; } + [JsonProperty("size", DefaultValueHandling = DefaultValueHandling.Ignore)] + public string Size { get; internal set; } [Preserve] - [JsonIgnore] - public Texture2D Image { get; internal set; } + [JsonProperty("quality", DefaultValueHandling = DefaultValueHandling.Ignore)] + public string Quality { get; internal set; } + + /// + /// Loads the image result as a . + /// + /// Optional, enable debug logging. + /// Optional, . + /// . + [Preserve] + public async Task LoadTextureAsync(bool debug = false, CancellationToken cancellationToken = default) + { + var image64 = string.IsNullOrWhiteSpace(Result) ? PartialImageResult : Result; + if (string.IsNullOrWhiteSpace(image64)) { return null; } + var (texture, _) = await TextureExtensions.ConvertFromBase64Async(image64, debug, cancellationToken); + return texture; + } } } diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Responses/ImageGenerationTool.cs b/OpenAI/Packages/com.openai.unity/Runtime/Responses/ImageGenerationTool.cs index 74b24922..3fb86610 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Responses/ImageGenerationTool.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Responses/ImageGenerationTool.cs @@ -19,6 +19,7 @@ public sealed class ImageGenerationTool : ITool public ImageGenerationTool( Model model = null, string background = null, + string inputFidelity = null, InputImageMask inputImageMask = null, string moderation = null, int? outputCompression = null, @@ -29,6 +30,7 @@ public ImageGenerationTool( { Model = string.IsNullOrWhiteSpace(model?.Id) ? Models.Model.GPT_Image_1 : model; Background = background; + InputFidelity = inputFidelity; InputImageMask = inputImageMask; Moderation = moderation; OutputCompression = outputCompression; @@ -43,6 +45,7 @@ public ImageGenerationTool( internal ImageGenerationTool( [JsonProperty("type")] string type, [JsonProperty("background")] string background, + [JsonProperty("input_fidelity")] string inputFidelity, [JsonProperty("input_image_mask")] InputImageMask inputImageMask, [JsonProperty("model")] string model, [JsonProperty("moderation")] string moderation, @@ -54,6 +57,7 @@ internal ImageGenerationTool( { Type = type; Background = background; + InputFidelity = inputFidelity; InputImageMask = inputImageMask; Model = model; Moderation = moderation; @@ -75,6 +79,15 @@ internal ImageGenerationTool( [JsonProperty("background", DefaultValueHandling = DefaultValueHandling.Ignore)] public string Background { get; } + /// + /// Control how much effort the model will exert to match the style and features, especially facial features, of input images. + /// This parameter is only supported for gpt-image-1.
+ /// Supports high and low. Defaults to low. + /// + [Preserve] + [JsonProperty("input_fidelity", DefaultValueHandling = DefaultValueHandling.Ignore)] + public string InputFidelity { get; } + /// /// Optional mask for inpainting. Contains image_url (string, optional) and file_id (string, optional). /// diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Responses/Response.cs b/OpenAI/Packages/com.openai.unity/Runtime/Responses/Response.cs index c71be352..5da85943 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Responses/Response.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Responses/Response.cs @@ -365,25 +365,6 @@ internal void InsertOutputItem(IResponseItem item, int index) output.Insert(index, item); } - [Preserve] - internal void UpdateOutputItem(IResponseItem item, int index) - { - if (item == null) - { - throw new ArgumentNullException(nameof(item)); - } - - if (index >= output.Count) - { - for (var i = output.Count; i <= index; i++) - { - output.Add(null); - } - } - - output[index] = item; - } - [Preserve] public void PrintUsage() { diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Responses/ResponsesEndpoint.cs b/OpenAI/Packages/com.openai.unity/Runtime/Responses/ResponsesEndpoint.cs index e9be2627..68d5fe15 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Responses/ResponsesEndpoint.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Responses/ResponsesEndpoint.cs @@ -134,13 +134,13 @@ private async Task StreamResponseAsync(string endpoint, string payload { IServerSentEvent serverSentEvent = null; var @event = ssEvent.Value.Value(); + var @object = ssEvent.Data ?? ssEvent.Value; if (EnableDebug) { - Debug.Log(ssEvent.ToJsonString()); + Debug.Log(@object.ToString(Formatting.None)); } - var @object = ssEvent.Data ?? ssEvent.Value; var text = @object["text"]?.Value(); var delta = @object["delta"]?.Value(); var itemId = @object["item_id"]?.Value(); @@ -161,20 +161,13 @@ private async Task StreamResponseAsync(string endpoint, string payload { var partialResponse = sseResponse.Deserialize(@object["response"], client); - if (response == null) + if (response == null || response.Id == partialResponse.Id) { response = partialResponse; } else { - if (response.Id == partialResponse.Id) - { - response = partialResponse; - } - else - { - throw new InvalidOperationException($"Response ID mismatch! Expected: {response.Id}, got: {partialResponse.Id}"); - } + throw new InvalidOperationException($"Response ID mismatch! Expected: {response.Id}, got: {partialResponse.Id}"); } serverSentEvent = response; @@ -197,36 +190,18 @@ private async Task StreamResponseAsync(string endpoint, string payload { serverSentEvent = part; } + break; } case "response.output_item.added": case "response.output_item.done": { var item = sseResponse.Deserialize(@object["item"], client); + response!.InsertOutputItem(item, outputIndex!.Value); if (@event == "response.output_item.done") { - if (item.Type == ResponseItemType.ImageGenerationCall && - item is ImageGenerationCall imageGenerationCall) - { - if (!string.IsNullOrWhiteSpace(imageGenerationCall.Result)) - { - var (texture, _) = await TextureExtensions.ConvertFromBase64Async(imageGenerationCall.Result, EnableDebug, cancellationToken); - imageGenerationCall.Image = texture; - } - - response!.UpdateOutputItem(imageGenerationCall, outputIndex!.Value); - serverSentEvent = imageGenerationCall; - } - else - { - response!.UpdateOutputItem(item, outputIndex!.Value); - serverSentEvent = item; - } - } - else // response.output_item.added - { - response!.InsertOutputItem(item, outputIndex!.Value); + serverSentEvent = item; } break; } @@ -244,11 +219,13 @@ private async Task StreamResponseAsync(string endpoint, string payload { functionToolCall.Delta = delta; } + break; } case "response.image_generation_call.in_progress": case "response.image_generation_call.generating": case "response.image_generation_call.partial_image": + case "response.image_generation_call.completed": { var imageGenerationCall = (ImageGenerationCall)response!.Output[outputIndex!.Value]; @@ -257,11 +234,13 @@ private async Task StreamResponseAsync(string endpoint, string payload throw new InvalidOperationException($"ImageGenerationCall ID mismatch! Expected: {imageGenerationCall.Id}, got: {itemId}"); } - //if (!string.IsNullOrWhiteSpace(imageGenerationCall.PartialImageResult)) - //{ - // var (texture, _) = await TextureExtensions.ConvertFromBase64Async(imageGenerationCall.PartialImageResult, EnableDebug, cancellationToken); - // imageGenerationCall.Image = texture; - //} + imageGenerationCall.Size = @object["size"]?.Value(); + imageGenerationCall.Quality = @object["quality"]?.Value(); + imageGenerationCall.Background = @object["background"]?.Value(); + imageGenerationCall.OutputFormat = @object["output_format"]?.Value(); + imageGenerationCall.RevisedPrompt = @object["revised_prompt"]?.Value(); + imageGenerationCall.PartialImageIndex = @object["partial_image_index"]?.Value(); + imageGenerationCall.PartialImageResult = @object["partial_image_b64"]?.Value(); serverSentEvent = imageGenerationCall; break; @@ -298,18 +277,15 @@ private async Task StreamResponseAsync(string endpoint, string payload partialContent = new AudioContent(audioContent.Type, base64Data: delta); audioContent.AppendFrom(partialContent); serverSentEvent = partialContent; - break; case "response.audio.transcript.delta": partialContent = new AudioContent(audioContent.Type, transcript: delta); audioContent.AppendFrom(partialContent); serverSentEvent = partialContent; - break; case "response.audio.done": case "response.audio.transcript.done": serverSentEvent = audioContent; - break; default: throw new InvalidOperationException($"Unexpected event type: {@event} for AudioContent."); @@ -333,7 +309,6 @@ private async Task StreamResponseAsync(string endpoint, string payload } serverSentEvent = textContent; - break; case RefusalContent refusalContent: var refusal = @object["refusal"]?.Value(); @@ -349,7 +324,6 @@ private async Task StreamResponseAsync(string endpoint, string payload } serverSentEvent = refusalContent; - break; case ReasoningContent reasoningContent: if (!string.IsNullOrWhiteSpace(text)) @@ -369,6 +343,7 @@ private async Task StreamResponseAsync(string endpoint, string payload } case "response.reasoning_summary_part.added": case "response.reasoning_summary_part.done": + { var summaryIndex = @object["summary_index"]!.Value(); var reasoningItem = (ReasoningItem)response!.Output[outputIndex!.Value]; var summaryItem = sseResponse.Deserialize(@object["part"], client); @@ -380,10 +355,12 @@ private async Task StreamResponseAsync(string endpoint, string payload } break; + } case "response.reasoning_text.delta": case "response.reasoning_text.done": + { var reasoningContentIndex = @object["content_index"]!.Value(); - reasoningItem = (ReasoningItem)response!.Output[outputIndex!.Value]; + var reasoningItem = (ReasoningItem)response!.Output[outputIndex!.Value]; var reasoningContentItem = reasoningItem.Content[reasoningContentIndex]; if (!string.IsNullOrWhiteSpace(text)) @@ -394,9 +371,12 @@ private async Task StreamResponseAsync(string endpoint, string payload reasoningContentItem.Delta = !string.IsNullOrWhiteSpace(delta) ? delta : null; serverSentEvent = reasoningContentItem; break; + } case "error": + { serverSentEvent = sseResponse.Deserialize(client); break; + } // TODO - implement handling for these events: case "response.code_interpreter_call.interpreting": case "response.code_interpreter_call.in_progress": @@ -408,7 +388,6 @@ private async Task StreamResponseAsync(string endpoint, string payload case "response.file_search_call.in_progress": case "response.file_search_call.searching": case "response.file_search_call.completed": - case "response.image_generation_call.completed": case "response.mcp_call_arguments.delta": case "response.mcp_call_arguments.done": case "response.mcp_call.in_progress": @@ -421,9 +400,11 @@ private async Task StreamResponseAsync(string endpoint, string payload case "response.web_search_call.searching": case "response.web_search_call.completed": default: + { // if not properly handled raise it up to caller to deal with it themselves. serverSentEvent = ssEvent; break; + } } } catch (Exception e) @@ -439,9 +420,7 @@ private async Task StreamResponseAsync(string endpoint, string payload }, new RestParameters(client.DefaultRequestHeaders), cancellationToken); // ReSharper restore AccessToModifiedClosure streamResponse.Validate(EnableDebug); - if (response == null) { return null; } - response = await response.WaitForStatusChangeAsync(timeout: -1, cancellationToken: cancellationToken); response.SetResponseData(streamResponse, client); return response; diff --git a/OpenAI/Packages/com.openai.unity/Samples~/Responses/ResponsesBehaviour.cs b/OpenAI/Packages/com.openai.unity/Samples~/Responses/ResponsesBehaviour.cs index f6cc5ea3..259d8363 100644 --- a/OpenAI/Packages/com.openai.unity/Samples~/Responses/ResponsesBehaviour.cs +++ b/OpenAI/Packages/com.openai.unity/Samples~/Responses/ResponsesBehaviour.cs @@ -147,9 +147,10 @@ async Task StreamEventHandler(string eventName, IServerSentEvent sseEvent) await GenerateSpeechAsync(message, destroyCancellationToken); break; case ImageGenerationCall imageGenerationCall: - if (imageGenerationCall.Image != null) + if (!string.IsNullOrWhiteSpace(imageGenerationCall.Result)) { - AddNewImageContent(imageGenerationCall.Image); + var image = await imageGenerationCall.LoadTextureAsync(enableDebug, destroyCancellationToken); + AddNewImageContent(image); scrollView.verticalNormalizedPosition = 0f; } break; diff --git a/OpenAI/Packages/com.openai.unity/Tests/OpenAI.Tests.asmdef b/OpenAI/Packages/com.openai.unity/Tests/OpenAI.Tests.asmdef index c5678797..9eeb27ef 100644 --- a/OpenAI/Packages/com.openai.unity/Tests/OpenAI.Tests.asmdef +++ b/OpenAI/Packages/com.openai.unity/Tests/OpenAI.Tests.asmdef @@ -2,9 +2,10 @@ "name": "OpenAI.Tests", "rootNamespace": "OpenAI.Tests", "references": [ + "GUID:3248779d86bd31747b5d2214f30b01ac", "GUID:27619889b8ba8c24980f49ee34dbb44a", "GUID:0acc523941302664db1f4e527237feb3", - "GUID:3248779d86bd31747b5d2214f30b01ac", + "GUID:6ab1da3c58a70364e92dc36aaec78f43", "GUID:7958db66189566541a6363568aee1575" ], "includePlatforms": [ diff --git a/OpenAI/Packages/com.openai.unity/Tests/TestFixture_14_Responses.cs b/OpenAI/Packages/com.openai.unity/Tests/TestFixture_14_Responses.cs index 0eaedeba..8dab1eb8 100644 --- a/OpenAI/Packages/com.openai.unity/Tests/TestFixture_14_Responses.cs +++ b/OpenAI/Packages/com.openai.unity/Tests/TestFixture_14_Responses.cs @@ -12,6 +12,7 @@ using System.Threading.Tasks; using UnityEngine; using Utilities.WebRequestRest.Interfaces; +using Utilities.Extensions; namespace OpenAI.Tests { @@ -79,7 +80,8 @@ public async Task Test_01_02_SimpleTestInput_Streaming() var response = await OpenAIClient.ResponsesEndpoint.CreateModelResponseAsync("Tell me a three sentence bedtime story about a unicorn.", async (@event, sseEvent) => { - Debug.Log($"{@event}:{sseEvent.ToJsonString()}"); + Assert.NotNull(@event); + Assert.NotNull(sseEvent); await Task.CompletedTask; }); @@ -361,88 +363,107 @@ async Task StreamCallback(string @event, IServerSentEvent sseEvent) catch (Exception e) { Debug.LogException(e); + throw; } } + [Test] public async Task Test_04_01_JsonSchema() { - Assert.IsNotNull(OpenAIClient.ResponsesEndpoint); + try + { + Assert.IsNotNull(OpenAIClient.ResponsesEndpoint); - var messages = new List + var messages = new List { new Message(Role.System, "You are a helpful math tutor. Guide the user through the solution step by step."), new Message(Role.User, "how can I solve 8x + 7 = -23") }; - var request = new CreateResponseRequest(messages, model: Model.GPT4_1_Nano); - var (mathResponse, response) = await OpenAIClient.ResponsesEndpoint.CreateModelResponseAsync(request); - Assert.NotNull(response); - Assert.IsNotEmpty(response.Id); - Assert.AreEqual(ResponseStatus.Completed, response.Status); - Assert.NotNull(mathResponse); - Assert.IsNotEmpty(mathResponse.Steps); + var request = new CreateResponseRequest(messages, model: Model.GPT4_1_Nano); + var (mathResponse, response) = await OpenAIClient.ResponsesEndpoint.CreateModelResponseAsync(request); + Assert.NotNull(response); + Assert.IsNotEmpty(response.Id); + Assert.AreEqual(ResponseStatus.Completed, response.Status); + Assert.NotNull(mathResponse); + Assert.IsNotEmpty(mathResponse.Steps); + + for (var i = 0; i < mathResponse.Steps.Count; i++) + { + var step = mathResponse.Steps[i]; + Assert.IsNotNull(step.Explanation); + Debug.Log($"Step {i}: {step.Explanation}"); + Assert.IsNotNull(step.Output); + Debug.Log($"Result: {step.Output}"); + } - for (var i = 0; i < mathResponse.Steps.Count; i++) + Assert.IsNotNull(mathResponse.FinalAnswer); + Debug.Log($"Final Answer: {mathResponse.FinalAnswer}"); + response.PrintUsage(); + } + catch (Exception e) { - var step = mathResponse.Steps[i]; - Assert.IsNotNull(step.Explanation); - Debug.Log($"Step {i}: {step.Explanation}"); - Assert.IsNotNull(step.Output); - Debug.Log($"Result: {step.Output}"); + Debug.LogException(e); + throw; } - - Assert.IsNotNull(mathResponse.FinalAnswer); - Debug.Log($"Final Answer: {mathResponse.FinalAnswer}"); - response.PrintUsage(); } [Test] public async Task Test_04_02_JsonSchema_Streaming() { - Assert.IsNotNull(OpenAIClient.ResponsesEndpoint); + try + { + Assert.IsNotNull(OpenAIClient.ResponsesEndpoint); - var messages = new List + var messages = new List { new Message(Role.System, "You are a helpful math tutor. Guide the user through the solution step by step."), new Message(Role.User, "how can I solve 8x + 7 = -23") }; - Task StreamCallback(string @event, IServerSentEvent sseEvent) - { - switch (sseEvent) + Task StreamCallback(string @event, IServerSentEvent sseEvent) { - case Message messageItem: - Assert.NotNull(messageItem); - var matchSchema = messageItem.FromSchema(); - Assert.NotNull(matchSchema); - Assert.IsNotEmpty(matchSchema.Steps); - - for (var i = 0; i < matchSchema.Steps.Count; i++) - { - var step = matchSchema.Steps[i]; - Assert.IsNotNull(step.Explanation); - Debug.Log($"Step {i}: {step.Explanation}"); - Assert.IsNotNull(step.Output); - Debug.Log($"Result: {step.Output}"); - } - - Assert.IsNotNull(matchSchema.FinalAnswer); - Debug.Log($"Final Answer: {matchSchema.FinalAnswer}"); - break; + switch (sseEvent) + { + case Message messageItem: + Assert.NotNull(messageItem); + var matchSchema = messageItem.FromSchema(); + Assert.NotNull(matchSchema); + Assert.IsNotEmpty(matchSchema.Steps); + + for (var i = 0; i < matchSchema.Steps.Count; i++) + { + var step = matchSchema.Steps[i]; + Assert.IsNotNull(step.Explanation); + Debug.Log($"Step {i}: {step.Explanation}"); + Assert.IsNotNull(step.Output); + Debug.Log($"Result: {step.Output}"); + } + + Assert.IsNotNull(matchSchema.FinalAnswer); + Debug.Log($"Final Answer: {matchSchema.FinalAnswer}"); + + break; + } + + return Task.CompletedTask; } - return Task.CompletedTask; + var request = new CreateResponseRequest(messages, model: Model.GPT4_1_Nano); + var (mathResponse, response) = await OpenAIClient.ResponsesEndpoint.CreateModelResponseAsync(request, StreamCallback); + Assert.NotNull(response); + Assert.IsNotEmpty(response.Id); + Assert.AreEqual(ResponseStatus.Completed, response.Status); + Assert.NotNull(mathResponse); + Assert.IsNotEmpty(mathResponse.Steps); + response.PrintUsage(); + } + catch (Exception e) + { + Debug.LogException(e); + throw; } - - var request = new CreateResponseRequest(messages, model: Model.GPT4_1_Nano); - var (mathResponse, response) = await OpenAIClient.ResponsesEndpoint.CreateModelResponseAsync(request, StreamCallback); - Assert.NotNull(response); - Assert.IsNotEmpty(response.Id); - Assert.AreEqual(ResponseStatus.Completed, response.Status); - Assert.NotNull(mathResponse); - Assert.IsNotEmpty(mathResponse.Steps); - response.PrintUsage(); } [Test] @@ -456,16 +477,19 @@ public async Task Test_05_01_Prompt() { new Message(Role.User, "What's the weather like today?"), }; + var tools = new List { Tool.GetOrCreateTool(typeof(WeatherService), nameof(WeatherService.GetCurrentWeatherAsync)), }; + var request = new CreateResponseRequest( input: conversation, model: Model.GPT4_1_Nano, prompt: new Prompt("pmpt_685c102c61608193b3654325fa76fc880b22337c811a3a71"), tools: tools, toolChoice: "none"); + var response = await OpenAIClient.ResponsesEndpoint.CreateModelResponseAsync(request); Assert.NotNull(response); Assert.IsNotEmpty(response.Id); @@ -531,5 +555,51 @@ public async Task Test_05_01_Prompt() throw; } } + + [Test] + public async Task Test_06_01_ImageGenerationTool() + { + try + { + Assert.NotNull(OpenAIClient.ResponsesEndpoint); + var tools = new List + { + new ImageGenerationTool( + model: Model.GPT_Image_1, + size: "1024x1024", + quality: "low", + outputFormat: "png") + }; + var request = new CreateResponseRequest( + input: new Message(Role.User, "Create an image of a futuristic city with flying cars."), + model: Model.GPT4_1_Nano, + tools: tools, + toolChoice: "auto"); + var response = await OpenAIClient.ResponsesEndpoint.CreateModelResponseAsync(request, async serverSentEvent => + { + if (serverSentEvent is ImageGenerationCall { Status: ResponseStatus.Generating } imageGenerationCall) + { + var image = await imageGenerationCall.LoadTextureAsync(debug: true); + Assert.NotNull(image); + image.Destroy(); + } + }); + Assert.NotNull(response); + Assert.IsNotEmpty(response.Id); + Assert.AreEqual(ResponseStatus.Completed, response.Status); + + // make sure we have at least the image generation call in the response output array + var imageCall = response.Output.FirstOrDefault(i => i.Type == ResponseItemType.ImageGenerationCall) as ImageGenerationCall; + Assert.NotNull(imageCall); + Assert.AreEqual(ResponseStatus.Generating, imageCall.Status); + + response.PrintUsage(); + } + catch (Exception e) + { + Debug.LogException(e); + throw; + } + } } } From 58e151ea7584df0ecc6c26954678ace7ca91dd6a Mon Sep 17 00:00:00 2001 From: Stephen Hodgson Date: Thu, 2 Oct 2025 21:28:10 -0400 Subject: [PATCH 18/22] completely rework and refactor the responses streaming event handling --- .../Responses/CodeInterpreterToolCall.cs | 26 +++- .../Runtime/Responses/CustomToolCall.cs | 22 ++- .../Runtime/Responses/FunctionToolCall.cs | 12 +- .../Runtime/Responses/MCPTool.cs | 4 +- .../Runtime/Responses/MCPToolCall.cs | 41 +++++- .../Runtime/Responses/Message.cs | 2 +- .../Runtime/Responses/ReasoningContent.cs | 20 ++- .../Runtime/Responses/ReasoningSummary.cs | 20 ++- .../Runtime/Responses/RefusalContent.cs | 24 +++- .../Runtime/Responses/ResponsesEndpoint.cs | 135 +++++++++++------- .../Runtime/Responses/TextContent.cs | 20 ++- .../Tests/TestFixture_14_Responses.cs | 86 +++++++++-- 12 files changed, 328 insertions(+), 84 deletions(-) diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Responses/CodeInterpreterToolCall.cs b/OpenAI/Packages/com.openai.unity/Runtime/Responses/CodeInterpreterToolCall.cs index 8670fc0d..2a14cc18 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Responses/CodeInterpreterToolCall.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Responses/CodeInterpreterToolCall.cs @@ -58,7 +58,27 @@ internal CodeInterpreterToolCall( /// [Preserve] [JsonProperty("code")] - public string Code { get; } + public string Code { get; internal set; } + + private string delta; + + [Preserve] + [JsonIgnore] + public string Delta + { + get => delta; + internal set + { + if (value == null) + { + delta = null; + } + else + { + delta += value; + } + } + } /// /// The results of the code interpreter tool call. @@ -73,5 +93,9 @@ internal CodeInterpreterToolCall( [Preserve] [JsonProperty("container_id", DefaultValueHandling = DefaultValueHandling.Ignore)] public string ContainerId { get; } + + [Preserve] + public override string ToString() + => Delta ?? Code ?? string.Empty; } } diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Responses/CustomToolCall.cs b/OpenAI/Packages/com.openai.unity/Runtime/Responses/CustomToolCall.cs index 1bc6a1f2..732bce23 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Responses/CustomToolCall.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Responses/CustomToolCall.cs @@ -80,6 +80,26 @@ internal CustomToolCall( /// [Preserve] [JsonProperty("input")] - public string Input { get; } + public string Input { get; internal set; } + + private string delta; + + [Preserve] + [JsonIgnore] + public string Delta + { + get => delta; + internal set + { + if (value == null) + { + delta = null; + } + else + { + delta += value; + } + } + } } } diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Responses/FunctionToolCall.cs b/OpenAI/Packages/com.openai.unity/Runtime/Responses/FunctionToolCall.cs index 54f116bd..ddb541cb 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Responses/FunctionToolCall.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Responses/FunctionToolCall.cs @@ -110,7 +110,17 @@ public JToken Arguments [JsonIgnore] internal string Delta { - set => argumentsString += value; + set + { + if (value == null) + { + argumentsString = null; + } + else + { + argumentsString += value; + } + } } [Preserve] diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Responses/MCPTool.cs b/OpenAI/Packages/com.openai.unity/Runtime/Responses/MCPTool.cs index 5b4e935b..36b1e544 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Responses/MCPTool.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Responses/MCPTool.cs @@ -35,7 +35,7 @@ public MCPTool( serverDescription: serverDescription, allowedTools: allowedTools, headers: headers, - requireApproval: requireApproval.ToString().ToLower()) + requireApproval: (object)requireApproval) { } @@ -79,7 +79,7 @@ public MCPTool( ServerDescription = serverDescription; AllowedTools = allowedTools; Headers = headers ?? new Dictionary(); - RequireApproval = requireApproval; + RequireApproval = requireApproval is MCPToolRequireApproval ? requireApproval.ToString().ToLower() : requireApproval; } [Preserve] diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Responses/MCPToolCall.cs b/OpenAI/Packages/com.openai.unity/Runtime/Responses/MCPToolCall.cs index 8bbd650e..67543296 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Responses/MCPToolCall.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Responses/MCPToolCall.cs @@ -69,12 +69,51 @@ internal MCPToolCall( [JsonProperty("server_label")] public string ServerLabel { get; } + private string argumentsString; + + private JToken arguments; + /// /// A JSON string of the arguments to pass to the function. /// [Preserve] [JsonProperty("arguments")] - public JToken Arguments { get; } + public JToken Arguments + { + get + { + if (arguments == null) + { + if (!string.IsNullOrWhiteSpace(argumentsString)) + { + arguments = JToken.FromObject(argumentsString, OpenAIClient.JsonSerializer); + } + else + { + arguments = null; + } + } + + return arguments; + } + internal set => arguments = value; + } + + [JsonIgnore] + internal string Delta + { + set + { + if (value == null) + { + argumentsString = null; + } + else + { + argumentsString += value; + } + } + } /// /// The output from the tool call. diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Responses/Message.cs b/OpenAI/Packages/com.openai.unity/Runtime/Responses/Message.cs index e9137c4d..19870a4d 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Responses/Message.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Responses/Message.cs @@ -106,7 +106,7 @@ public IReadOnlyList Content } [Preserve] - internal void AddContentItem(IResponseContent item, int index) + internal void AddOrUpdateContentItem(IResponseContent item, int index) { if (item == null) { diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Responses/ReasoningContent.cs b/OpenAI/Packages/com.openai.unity/Runtime/Responses/ReasoningContent.cs index 152c3d73..98485391 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Responses/ReasoningContent.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Responses/ReasoningContent.cs @@ -32,9 +32,25 @@ internal ReasoningContent( [JsonProperty("text")] public string Text { get; internal set; } + private string delta; + [Preserve] [JsonIgnore] - public string Delta { get; internal set; } + public string Delta + { + get => delta; + internal set + { + if (value == null) + { + delta = null; + } + else + { + delta += value; + } + } + } [Preserve] [JsonIgnore] @@ -42,6 +58,6 @@ internal ReasoningContent( [Preserve] public override string ToString() - => string.IsNullOrWhiteSpace(Text) ? Delta : Text; + => Delta ?? Text ?? string.Empty; } } diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Responses/ReasoningSummary.cs b/OpenAI/Packages/com.openai.unity/Runtime/Responses/ReasoningSummary.cs index 4f34ebd8..44cad918 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Responses/ReasoningSummary.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Responses/ReasoningSummary.cs @@ -30,9 +30,25 @@ internal ReasoningSummary( [JsonProperty("text")] public string Text { get; internal set; } + private string delta; + [Preserve] [JsonIgnore] - public string Delta { get; internal set; } + public string Delta + { + get => delta; + internal set + { + if (value == null) + { + delta = null; + } + else + { + delta += value; + } + } + } [Preserve] [JsonIgnore] @@ -45,6 +61,6 @@ public string ToJsonString() [Preserve] public override string ToString() - => string.IsNullOrWhiteSpace(Text) ? Delta : Text; + => Delta ?? Text ?? string.Empty; } } diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Responses/RefusalContent.cs b/OpenAI/Packages/com.openai.unity/Runtime/Responses/RefusalContent.cs index 03142412..207548a8 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Responses/RefusalContent.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Responses/RefusalContent.cs @@ -32,9 +32,25 @@ internal RefusalContent( [JsonProperty("refusal")] public string Refusal { get; internal set; } + private string delta; + [Preserve] [JsonIgnore] - public string Delta { get; internal set; } + public string Delta + { + get => delta; + internal set + { + if (value == null) + { + delta = null; + } + else + { + delta += value; + } + } + } [Preserve] [JsonIgnore] @@ -42,10 +58,6 @@ internal RefusalContent( [Preserve] public override string ToString() - => !string.IsNullOrWhiteSpace(Refusal) - ? Refusal - : !string.IsNullOrWhiteSpace(Delta) - ? Delta - : string.Empty; + => Delta ?? Refusal ?? string.Empty; } } diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Responses/ResponsesEndpoint.cs b/OpenAI/Packages/com.openai.unity/Runtime/Responses/ResponsesEndpoint.cs index 68d5fe15..8d315e08 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Responses/ResponsesEndpoint.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Responses/ResponsesEndpoint.cs @@ -184,7 +184,7 @@ private async Task StreamResponseAsync(string endpoint, string payload throw new InvalidOperationException($"MessageItem ID mismatch! Expected: {messageItem.Id}, got: {itemId}"); } - messageItem.AddContentItem(part, contentIndex!.Value); + messageItem.AddOrUpdateContentItem(part, contentIndex!.Value); if (@event == "response.content_part.done") { @@ -203,6 +203,7 @@ private async Task StreamResponseAsync(string endpoint, string payload { serverSentEvent = item; } + break; } case "response.function_call_arguments.delta": @@ -215,11 +216,22 @@ private async Task StreamResponseAsync(string endpoint, string payload throw new InvalidOperationException($"FunctionToolCall ID mismatch! Expected: {functionToolCall.Id}, got: {itemId}"); } - if (!string.IsNullOrWhiteSpace(delta)) + functionToolCall.Delta = delta; + functionToolCall.Arguments = @object["arguments"]; + break; + } + case "response.custom_tool_call_input.delta": + case "response.custom_tool_call_input.done": + { + var customToolCall = (CustomToolCall)response!.Output[outputIndex!.Value]; + + if (customToolCall.Id != itemId) { - functionToolCall.Delta = delta; + throw new InvalidOperationException($"CustomToolCall ID mismatch! Expected: {customToolCall.Id}, got: {itemId}"); } + customToolCall.Delta = delta; + customToolCall.Input = @object["input"]?.Value(); break; } case "response.image_generation_call.in_progress": @@ -242,6 +254,7 @@ private async Task StreamResponseAsync(string endpoint, string payload imageGenerationCall.PartialImageIndex = @object["partial_image_index"]?.Value(); imageGenerationCall.PartialImageResult = @object["partial_image_b64"]?.Value(); + response!.InsertOutputItem(imageGenerationCall, outputIndex!.Value); serverSentEvent = imageGenerationCall; break; } @@ -254,8 +267,8 @@ private async Task StreamResponseAsync(string endpoint, string payload case "response.output_text.done": case "response.refusal.delta": case "response.refusal.done": - case "response.reasoning_summary_text.delta": - case "response.reasoning_summary_text.done": + case "response.reasoning_text.delta": + case "response.reasoning_text.done": { var messageItem = (Message)response!.Output[outputIndex!.Value]; @@ -276,11 +289,13 @@ private async Task StreamResponseAsync(string endpoint, string payload case "response.audio.delta": partialContent = new AudioContent(audioContent.Type, base64Data: delta); audioContent.AppendFrom(partialContent); + messageItem.AddOrUpdateContentItem(audioContent, contentIndex!.Value); serverSentEvent = partialContent; break; case "response.audio.transcript.delta": partialContent = new AudioContent(audioContent.Type, transcript: delta); audioContent.AppendFrom(partialContent); + messageItem.AddOrUpdateContentItem(audioContent, contentIndex!.Value); serverSentEvent = partialContent; break; case "response.audio.done": @@ -293,13 +308,6 @@ private async Task StreamResponseAsync(string endpoint, string payload break; case TextContent textContent: - if (!string.IsNullOrWhiteSpace(text)) - { - textContent.Text = text; - } - - textContent.Delta = !string.IsNullOrWhiteSpace(delta) ? delta : null; - var annotationIndex = @object["annotation_index"]?.Value(); if (annotationIndex.HasValue) @@ -307,35 +315,26 @@ private async Task StreamResponseAsync(string endpoint, string payload var annotation = sseResponse.Deserialize(@object["annotation"], client); textContent.InsertAnnotation(annotation, annotationIndex.Value); } + else + { + textContent.Text = text; + textContent.Delta = delta; + } + messageItem.AddOrUpdateContentItem(textContent, contentIndex!.Value); serverSentEvent = textContent; break; case RefusalContent refusalContent: - var refusal = @object["refusal"]?.Value(); - - if (!string.IsNullOrWhiteSpace(refusal)) - { - refusalContent.Refusal = refusal; - } - - if (!string.IsNullOrWhiteSpace(delta)) - { - refusalContent.Delta = delta; - } - + refusalContent.Delta = delta; + refusalContent.Refusal = @object["refusal"]?.Value(); + messageItem.AddOrUpdateContentItem(refusalContent, contentIndex!.Value); serverSentEvent = refusalContent; break; case ReasoningContent reasoningContent: - if (!string.IsNullOrWhiteSpace(text)) - { - reasoningContent.Text = text; - } - - if (!string.IsNullOrWhiteSpace(delta)) - { - reasoningContent.Delta = delta; - } - + reasoningContent.Text = text; + reasoningContent.Delta = delta; + messageItem.AddOrUpdateContentItem(reasoningContent, contentIndex!.Value); + serverSentEvent = reasoningContent; break; } @@ -343,33 +342,65 @@ private async Task StreamResponseAsync(string endpoint, string payload } case "response.reasoning_summary_part.added": case "response.reasoning_summary_part.done": + case "response.reasoning_summary_text.delta": + case "response.reasoning_summary_text.done": { var summaryIndex = @object["summary_index"]!.Value(); var reasoningItem = (ReasoningItem)response!.Output[outputIndex!.Value]; - var summaryItem = sseResponse.Deserialize(@object["part"], client); - reasoningItem.InsertSummary(summaryItem, summaryIndex); - if (@event == "response.reasoning_summary_part.done") + if (reasoningItem.Id != itemId) + { + throw new InvalidOperationException($"ReasoningItem ID mismatch! Expected: {reasoningItem.Id}, got: {itemId}"); + } + + ReasoningSummary summaryItem; + + if (@object["part"] != null) + { + summaryItem = sseResponse.Deserialize(@object["part"], client); + reasoningItem.InsertSummary(summaryItem, summaryIndex); + } + else { - serverSentEvent = summaryItem; + summaryItem = reasoningItem.Summary[summaryIndex]; + summaryItem.Delta = delta; + summaryItem.Text = text; } + response!.InsertOutputItem(reasoningItem, outputIndex!.Value); + serverSentEvent = summaryItem; break; } - case "response.reasoning_text.delta": - case "response.reasoning_text.done": + case "response.mcp_call_arguments.delta": + case "response.mcp_call_arguments.done": { - var reasoningContentIndex = @object["content_index"]!.Value(); - var reasoningItem = (ReasoningItem)response!.Output[outputIndex!.Value]; - var reasoningContentItem = reasoningItem.Content[reasoningContentIndex]; + var mcpToolCall = (MCPToolCall)response!.Output[outputIndex!.Value]; - if (!string.IsNullOrWhiteSpace(text)) + if (mcpToolCall.Id != itemId) { - reasoningContentItem.Text = text; + throw new InvalidOperationException($"MCPToolCall ID mismatch! Expected: {mcpToolCall.Id}, got: {itemId}"); } - reasoningContentItem.Delta = !string.IsNullOrWhiteSpace(delta) ? delta : null; - serverSentEvent = reasoningContentItem; + mcpToolCall.Delta = delta; + mcpToolCall.Arguments = @object["arguments"]; + response!.InsertOutputItem(mcpToolCall, outputIndex!.Value); + serverSentEvent = mcpToolCall; + break; + } + case "response.code_interpreter_call_code.delta": + case "response.code_interpreter_call_code.done": + { + var codeInterpreterToolCall = (CodeInterpreterToolCall)response!.Output[outputIndex!.Value]; + + if (codeInterpreterToolCall.Id != itemId) + { + throw new InvalidOperationException($"CodeInterpreterToolCall ID mismatch! Expected: {codeInterpreterToolCall.Id}, got: {itemId}"); + } + + codeInterpreterToolCall.Delta = delta; + codeInterpreterToolCall.Code = @object["code"]?.Value(); + response!.InsertOutputItem(codeInterpreterToolCall, outputIndex!.Value); + serverSentEvent = codeInterpreterToolCall; break; } case "error": @@ -377,19 +408,13 @@ private async Task StreamResponseAsync(string endpoint, string payload serverSentEvent = sseResponse.Deserialize(client); break; } - // TODO - implement handling for these events: - case "response.code_interpreter_call.interpreting": + // Event status messages with no data payloads: case "response.code_interpreter_call.in_progress": + case "response.code_interpreter_call.interpreting": case "response.code_interpreter_call.completed": - case "response.code_interpreter_call_code.delta": - case "response.code_interpreter_call_code.done": - case "response.custom_tool_call_input.delta": - case "response.custom_tool_call_input.done": case "response.file_search_call.in_progress": case "response.file_search_call.searching": case "response.file_search_call.completed": - case "response.mcp_call_arguments.delta": - case "response.mcp_call_arguments.done": case "response.mcp_call.in_progress": case "response.mcp_call.completed": case "response.mcp_call.failed": diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Responses/TextContent.cs b/OpenAI/Packages/com.openai.unity/Runtime/Responses/TextContent.cs index 11684639..4d83acd2 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Responses/TextContent.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Responses/TextContent.cs @@ -57,9 +57,25 @@ public IReadOnlyList Annotations [JsonProperty("logprobs", DefaultValueHandling = DefaultValueHandling.Ignore)] public IReadOnlyList LogProbs { get; } + private string delta; + [Preserve] [JsonProperty("delta", DefaultValueHandling = DefaultValueHandling.Ignore)] - public string Delta { get; internal set; } + public string Delta + { + get => delta; + internal set + { + if (value == null) + { + delta = null; + } + else + { + delta += value; + } + } + } [JsonIgnore] public string Object => Type.ToString(); @@ -87,6 +103,6 @@ internal void InsertAnnotation(IAnnotation item, int index) [Preserve] public override string ToString() - => Text ?? string.Empty; + => Delta ?? Text ?? string.Empty; } } diff --git a/OpenAI/Packages/com.openai.unity/Tests/TestFixture_14_Responses.cs b/OpenAI/Packages/com.openai.unity/Tests/TestFixture_14_Responses.cs index 8dab1eb8..c7fc25f1 100644 --- a/OpenAI/Packages/com.openai.unity/Tests/TestFixture_14_Responses.cs +++ b/OpenAI/Packages/com.openai.unity/Tests/TestFixture_14_Responses.cs @@ -9,10 +9,11 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Threading.Tasks; using UnityEngine; -using Utilities.WebRequestRest.Interfaces; using Utilities.Extensions; +using Utilities.WebRequestRest.Interfaces; +using Message = OpenAI.Responses.Message; +using Task = System.Threading.Tasks.Task; namespace OpenAI.Tests { @@ -67,6 +68,7 @@ public async Task Test_01_01_SimpleTextInput() catch (Exception e) { Debug.LogException(e); + throw; } } @@ -105,6 +107,7 @@ public async Task Test_01_02_SimpleTestInput_Streaming() catch (Exception e) { Debug.LogException(e); + throw; } } @@ -190,6 +193,7 @@ public async Task Test_02_01_FunctionToolCall() catch (Exception e) { Debug.LogException(e); + throw; } } @@ -272,6 +276,7 @@ async Task StreamCallback(string @event, IServerSentEvent sseEvent) catch (Exception e) { Debug.LogException(e); + throw; } } @@ -307,6 +312,7 @@ public async Task Test_03_01_Reasoning() catch (Exception e) { Debug.LogException(e); + throw; } } @@ -376,10 +382,10 @@ public async Task Test_04_01_JsonSchema() Assert.IsNotNull(OpenAIClient.ResponsesEndpoint); var messages = new List - { - new Message(Role.System, "You are a helpful math tutor. Guide the user through the solution step by step."), - new Message(Role.User, "how can I solve 8x + 7 = -23") - }; + { + new Message(Role.System, "You are a helpful math tutor. Guide the user through the solution step by step."), + new Message(Role.User, "how can I solve 8x + 7 = -23") + }; var request = new CreateResponseRequest(messages, model: Model.GPT4_1_Nano); var (mathResponse, response) = await OpenAIClient.ResponsesEndpoint.CreateModelResponseAsync(request); @@ -405,6 +411,7 @@ public async Task Test_04_01_JsonSchema() catch (Exception e) { Debug.LogException(e); + throw; } } @@ -417,10 +424,10 @@ public async Task Test_04_02_JsonSchema_Streaming() Assert.IsNotNull(OpenAIClient.ResponsesEndpoint); var messages = new List - { - new Message(Role.System, "You are a helpful math tutor. Guide the user through the solution step by step."), - new Message(Role.User, "how can I solve 8x + 7 = -23") - }; + { + new Message(Role.System, "You are a helpful math tutor. Guide the user through the solution step by step."), + new Message(Role.User, "how can I solve 8x + 7 = -23") + }; Task StreamCallback(string @event, IServerSentEvent sseEvent) { @@ -462,6 +469,7 @@ Task StreamCallback(string @event, IServerSentEvent sseEvent) catch (Exception e) { Debug.LogException(e); + throw; } } @@ -552,6 +560,7 @@ public async Task Test_05_01_Prompt() catch (Exception e) { Debug.LogException(e); + throw; } } @@ -562,6 +571,7 @@ public async Task Test_06_01_ImageGenerationTool() try { Assert.NotNull(OpenAIClient.ResponsesEndpoint); + var tools = new List { new ImageGenerationTool( @@ -596,6 +606,62 @@ public async Task Test_06_01_ImageGenerationTool() response.PrintUsage(); } catch (Exception e) + { + Debug.LogException(e); + + throw; + } + } + + [Test] + public async Task Test_07_01_MCPTool() + { + try + { + Assert.NotNull(OpenAIClient.ResponsesEndpoint); + await Task.CompletedTask; + + var conversation = new List + { + new Message(Role.System, "You are a Dungeons and Dragons Master. Guide the players through the game turn by turn."), + new Message(Role.User, "Roll 2d4+1") + }; + var tools = new List + { + new MCPTool( + serverLabel: "dmcp", + serverDescription: "A Dungeons and Dragons MCP server to assist with dice rolling.", + serverUrl: "https://dmcp-server.deno.dev/sse", + requireApproval: MCPToolRequireApproval.Never) + }; + + Task StreamEventHandler(string @event, IServerSentEvent serverSentEvent) + { + switch (serverSentEvent) + { + case MCPListTools mcpListTools: + Assert.NotNull(mcpListTools); + Assert.IsNotEmpty(mcpListTools.Tools); + break; + case MCPToolCall mcpToolCall: + Assert.NotNull(mcpToolCall); + Assert.IsNotEmpty(mcpToolCall.Output); + break; + default: + throw new ArgumentOutOfRangeException(nameof(serverSentEvent)); + } + + return Task.CompletedTask; + } + + var request = new CreateResponseRequest(conversation, Model.GPT4_1_Nano, tools: tools, toolChoice: "auto"); + var response = await OpenAIClient.ResponsesEndpoint.CreateModelResponseAsync(request, StreamEventHandler); + + Assert.NotNull(response); + Assert.IsNotEmpty(response.Id); + Assert.AreEqual(ResponseStatus.Completed, response.Status); + } + catch (Exception e) { Debug.LogException(e); throw; From 062f8b96169a110554d5023c4891c74cf89ce02b Mon Sep 17 00:00:00 2001 From: Stephen Hodgson Date: Thu, 2 Oct 2025 23:29:37 -0400 Subject: [PATCH 19/22] missing server event setters cleanup tests --- .../com.openai.unity/Runtime/Responses/ResponsesEndpoint.cs | 4 ++++ .../com.openai.unity/Tests/TestFixture_14_Responses.cs | 5 ----- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Responses/ResponsesEndpoint.cs b/OpenAI/Packages/com.openai.unity/Runtime/Responses/ResponsesEndpoint.cs index 8d315e08..4fdc38e8 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Responses/ResponsesEndpoint.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Responses/ResponsesEndpoint.cs @@ -218,6 +218,8 @@ private async Task StreamResponseAsync(string endpoint, string payload functionToolCall.Delta = delta; functionToolCall.Arguments = @object["arguments"]; + response!.InsertOutputItem(functionToolCall, outputIndex!.Value); + serverSentEvent = functionToolCall; break; } case "response.custom_tool_call_input.delta": @@ -232,6 +234,8 @@ private async Task StreamResponseAsync(string endpoint, string payload customToolCall.Delta = delta; customToolCall.Input = @object["input"]?.Value(); + response!.InsertOutputItem(customToolCall, outputIndex!.Value); + serverSentEvent = customToolCall; break; } case "response.image_generation_call.in_progress": diff --git a/OpenAI/Packages/com.openai.unity/Tests/TestFixture_14_Responses.cs b/OpenAI/Packages/com.openai.unity/Tests/TestFixture_14_Responses.cs index c7fc25f1..db8966cf 100644 --- a/OpenAI/Packages/com.openai.unity/Tests/TestFixture_14_Responses.cs +++ b/OpenAI/Packages/com.openai.unity/Tests/TestFixture_14_Responses.cs @@ -608,7 +608,6 @@ public async Task Test_06_01_ImageGenerationTool() catch (Exception e) { Debug.LogException(e); - throw; } } @@ -641,14 +640,10 @@ Task StreamEventHandler(string @event, IServerSentEvent serverSentEvent) { case MCPListTools mcpListTools: Assert.NotNull(mcpListTools); - Assert.IsNotEmpty(mcpListTools.Tools); break; case MCPToolCall mcpToolCall: Assert.NotNull(mcpToolCall); - Assert.IsNotEmpty(mcpToolCall.Output); break; - default: - throw new ArgumentOutOfRangeException(nameof(serverSentEvent)); } return Task.CompletedTask; From 4baecb9ab4a12bdda37668cf5b43266b62239afd Mon Sep 17 00:00:00 2001 From: Stephen Hodgson Date: Thu, 2 Oct 2025 23:42:56 -0400 Subject: [PATCH 20/22] update build workflow --- .github/workflows/build-options.json | 43 +++++++++++ .github/workflows/unity.yml | 108 +++++++-------------------- 2 files changed, 72 insertions(+), 79 deletions(-) create mode 100644 .github/workflows/build-options.json diff --git a/.github/workflows/build-options.json b/.github/workflows/build-options.json new file mode 100644 index 00000000..17c7f4ff --- /dev/null +++ b/.github/workflows/build-options.json @@ -0,0 +1,43 @@ +{ + "os": [ + "ubuntu-latest", + "windows-latest", + "macos-latest" + ], + "unity-version": [ + "2022.x", + "6000.0.x", + "6000.1.x", + "6000.2.x" + ], + "include": [ + { + "os": "ubuntu-latest", + "build-target": "StandaloneLinux64" + }, + { + "os": "ubuntu-latest", + "build-target": "WebGL" + }, + { + "os": "ubuntu-latest", + "build-target": "Android" + }, + { + "os": "ubuntu-latest", + "build-target": "iOS" + }, + { + "os": "windows-latest", + "build-target": "StandaloneWindows64" + }, + { + "os": "windows-latest", + "build-target": "WSAPlayer" + }, + { + "os": "macos-latest", + "build-target": "StandaloneOSX" + } + ] +} \ No newline at end of file diff --git a/.github/workflows/unity.yml b/.github/workflows/unity.yml index 396d6829..a304c155 100644 --- a/.github/workflows/unity.yml +++ b/.github/workflows/unity.yml @@ -3,94 +3,44 @@ on: schedule: - cron: '0 0 * * 0' # Every Sunday at midnight push: - branches: - - 'main' + branches: [main] pull_request: types: [opened, reopened, synchronize, ready_for_review] - branches: - - '**' - workflow_dispatch: + branches: ['**'] + workflow_dispatch: # manual workflow trigger concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: ${{(github.event_name == 'pull_request' || github.event.action == 'synchronize')}} jobs: - build: - name: ${{ matrix.os }} ${{ matrix.unity-version }} ${{ matrix.build-target }} - runs-on: ${{ matrix.os }} + setup: if: github.event_name != 'pull_request' || !github.event.pull_request.draft + runs-on: ubuntu-latest permissions: contents: read - strategy: - fail-fast: false - matrix: - include: # for each os specify the build targets - - os: ubuntu-latest - unity-version: 2022.x - build-target: Android - - os: ubuntu-latest - unity-version: 6000.x - build-target: Android - - os: ubuntu-latest - unity-version: 2022.x - build-target: StandaloneLinux64 - - os: ubuntu-latest - unity-version: 6000.x - build-target: StandaloneLinux64 - - os: ubuntu-latest - unity-version: 2022.x - build-target: WebGL - - os: ubuntu-latest - unity-version: 6000.x - build-target: WebGL - - os: windows-latest - unity-version: 2022.x - build-target: StandaloneWindows64 - - os: windows-latest - unity-version: 6000.x - build-target: StandaloneWindows64 - - os: windows-latest - unity-version: 2022.x - build-target: WSAPlayer - - os: windows-latest - unity-version: 6000.x - build-target: WSAPlayer - - os: macos-latest - unity-version: 2022.x - build-target: iOS - - os: macos-latest - unity-version: 6000.x - build-target: iOS - - os: macos-latest - unity-version: 2022.x - build-target: StandaloneOSX - - os: macos-latest - unity-version: 6000.x - build-target: StandaloneOSX steps: - - uses: actions/checkout@v4 - - uses: RageAgainstThePixel/unity-setup@v1 - with: - unity-version: ${{ matrix.unity-version }} - build-targets: ${{ matrix.build-target }} - - uses: RageAgainstThePixel/activate-unity-license@v1 - with: - license: 'Personal' - username: ${{ secrets.UNITY_USERNAME }} - password: ${{ secrets.UNITY_PASSWORD }} - - uses: RageAgainstThePixel/unity-action@v1 - name: '${{ matrix.build-target }}-Validate' - with: - build-target: ${{ matrix.build-target }} - log-name: '${{ matrix.build-target }}-Validate' - args: '-quit -nographics -batchmode -executeMethod Utilities.Editor.BuildPipeline.UnityPlayerBuildTools.ValidateProject -importTMProEssentialsAsset' - - uses: RageAgainstThePixel/unity-action@v1 - name: '${{ matrix.build-target }}-Build' + - uses: actions/checkout@v5 with: - build-target: ${{ matrix.build-target }} - log-name: '${{ matrix.build-target }}-Build' - args: '-quit -nographics -batchmode -executeMethod Utilities.Editor.BuildPipeline.UnityPlayerBuildTools.StartCommandLineBuild' - - uses: actions/upload-artifact@v4 - if: success() || failure() + sparse-checkout: .github/ + - uses: RageAgainstThePixel/job-builder@v1 + id: setup-jobs with: - name: '${{ github.run_number }}.${{ github.run_attempt }}-${{ matrix.os }}-${{ matrix.unity-version }}-${{ matrix.build-target }}-Artifacts' - path: '${{ github.workspace }}/**/*.log' + build-options: ./.github/workflows/build-options.json + group-by: 'unity-version' + job-name-prefix: 'Build' + outputs: + jobs: ${{ steps.setup-jobs.outputs.jobs }} + validate: + if: ${{ needs.setup.outputs.jobs }} + needs: setup + name: ${{ matrix.jobs.name }} + permissions: + contents: read + actions: write + strategy: + matrix: ${{ fromJSON(needs.setup.outputs.jobs) }} + fail-fast: false + max-parallel: 1 + secrets: inherit + uses: RageAgainstThePixel/workflows/.github/workflows/build-unity-package.yml@main + with: + matrix: ${{ toJSON(matrix.jobs.matrix) }} \ No newline at end of file From 143b8bf9dd7a9f0715d42999c64ce4afdfd88e02 Mon Sep 17 00:00:00 2001 From: Stephen Hodgson Date: Thu, 2 Oct 2025 23:46:55 -0400 Subject: [PATCH 21/22] fix typos --- OpenAI/Packages/com.openai.unity/Runtime/Models/Model.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Models/Model.cs b/OpenAI/Packages/com.openai.unity/Runtime/Models/Model.cs index 7878c198..69544e5e 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Models/Model.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Models/Model.cs @@ -525,7 +525,7 @@ internal Model( #endregion Image Models - #region Specilized Models + #region Specialized Models /// /// GPT-5-Codex is a version of GPT-5 optimized for agentic coding tasks in Codex or similar environments. @@ -546,12 +546,12 @@ internal Model( /// public static Model Codex_Mini_Latest { get; } = new("codex-mini-latest", "openai"); - #endregion Specilized Models + #endregion Specialized Models #region Open Weight Models /// - /// gpt-oss-120bis our most powerful open-weight model, which fits into a single H100 GPU (117B parameters with 5.1B active parameters). + /// gpt-oss-120b is our most powerful open-weight model, which fits into a single H100 GPU (117B parameters with 5.1B active parameters). /// /// /// - Context Window: 131,072 context window
From 37229bf6b2525457e2b40e031f958ff09f5ca8c1 Mon Sep 17 00:00:00 2001 From: Stephen Hodgson Date: Fri, 3 Oct 2025 00:30:23 -0400 Subject: [PATCH 22/22] updated docs --- .../com.openai.unity/Documentation~/README.md | 142 +++++++++++++++++- .../com.openai.unity/Runtime/Models/Model.cs | 22 +-- .../Samples~/Responses/ResponsesBehaviour.cs | 2 +- README.md | 129 +++++++++++++++- 4 files changed, 272 insertions(+), 23 deletions(-) diff --git a/OpenAI/Packages/com.openai.unity/Documentation~/README.md b/OpenAI/Packages/com.openai.unity/Documentation~/README.md index f2ed0eff..aba1c752 100644 --- a/OpenAI/Packages/com.openai.unity/Documentation~/README.md +++ b/OpenAI/Packages/com.openai.unity/Documentation~/README.md @@ -80,6 +80,15 @@ openupm add com.openai.unity - [List Input Items](#list-input-items) - [Cancel Response](#cancel-response) - [Delete Response](#delete-response) +- [Conversations](#conversations) :new: + - [Create Conversation](#create-conversation) :new: + - [Retrieve Conversation](#retrieve-conversation) :new: + - [Update Conversation](#update-conversation) :new: + - [Delete Conversation](#delete-conversation) :new: + - [List Conversation Items](#list-conversation-items) :new: + - [Create Conversation Item](#create-conversation-item) :new: + - [Retrieve Conversation Item](#retrieve-conversation-item) :new: + - [Delete Conversation Item](#delete-conversation-item) :new: - [Realtime](#realtime) - [Create Realtime Session](#create-realtime-session) - [Client Events](#client-events) @@ -452,7 +461,7 @@ Creates a model response. Provide text or image inputs to generate text or JSON var api = new OpenAIClient(); var response = await api.ResponsesEndpoint.CreateModelResponseAsync("Tell me a three sentence bedtime story about a unicorn."); var responseItem = response.Output.LastOrDefault(); -Debug.Log($"{messageItem.Role}:{textContent.Text}"); +Debug.Log($"{responseItem.Role}:{responseItem}"); response.PrintUsage(); ``` @@ -469,7 +478,7 @@ var tools = new List { Tool.GetOrCreateTool(typeof(DateTimeUtility), nameof(DateTimeUtility.GetDateTime)) }; -var request = new CreateResponseRequest(conversation, Model.GPT4_1_Nano, tools: tools); +var request = new CreateResponseRequest(conversation, Model.GPT5_Nano, tools: tools); async Task StreamCallback(string @event, IServerSentEvent sseEvent) { @@ -482,7 +491,7 @@ async Task StreamCallback(string @event, IServerSentEvent sseEvent) conversation.Add(functionToolCall); var output = await functionToolCall.InvokeFunctionAsync(); conversation.Add(output); - await api.ResponsesEndpoint.CreateModelResponseAsync(new(conversation, Model.GPT4_1_Nano, tools: tools, toolChoice: "none"), StreamCallback); + await api.ResponsesEndpoint.CreateModelResponseAsync(new(conversation, Model.GPT5_Nano, tools: tools, toolChoice: "none"), StreamCallback); break; } } @@ -541,6 +550,120 @@ Assert.IsTrue(isDeleted); --- +### [Conversations](https://platform.openai.com/docs/api-reference/conversations) + +Create and manage conversations to store and retrieve conversation state across Response API calls. + +The Conversations API is accessed via `OpenAIClient.ConversationsEndpoint` + +#### [Create Conversation](https://platform.openai.com/docs/api-reference/conversations/create) + +Create a conversation. + +```csharp +var api = new OpenAIClient(); +conversation = await api.ConversationsEndpoint.CreateConversationAsync( + new CreateConversationRequest(new Message(Role.Developer, systemPrompt))); +Debug.Log(conversation.ToString()); +// use the conversation object when creating responses. +var request = await api.ResponsesEndpoint.CreateResponseAsync( + new CreateResponseRequest(textInput: "Hello!", conversationId: conversation, model: Model.GPT5_Nano)); +var response = await openAI.ResponsesEndpoint.CreateModelResponseAsync(request); +var responseItem = response.Output.LastOrDefault(); +Debug.Log($"{responseItem.Role}:{responseItem}"); +response.PrintUsage(); +``` + +#### [Retrieve Conversation](https://platform.openai.com/docs/api-reference/conversations/retrieve) + +Get a conversation by id. + +```csharp +var api = new OpenAIClient(); +var conversation = await api.ConversationsEndpoint.GetConversationAsync("conversation-id"); +Debug.Log(conversation.ToString()); +``` + +#### [Update Conversation](https://platform.openai.com/docs/api-reference/conversations/update) + +Update a conversation with custom metadata. + +```csharp +var api = new OpenAIClient(); +var metadata = new Dictionary +{ + { "favorite_color", "blue" }, + { "favorite_food", "pizza" } +}; +var updatedConversation = await api.ConversationsEndpoint.UpdateConversationAsync("conversation-id", metadata); +``` + +#### [Delete Conversation](https://platform.openai.com/docs/api-reference/conversations/delete) + +Delete a conversation by id. + +```csharp +var api = new OpenAIClient(); +var isDeleted = await api.ConversationsEndpoint.DeleteConversationAsync("conversation-id"); +Assert.IsTrue(isDeleted); +``` + +#### [List Conversation Items](https://platform.openai.com/docs/api-reference/conversations/list-items) + +List all items for a conversation with the given ID. + +```csharp +var api = new OpenAIClient(); +var query = new ListQuery(limit: 10); +var items = await api.ConversationsEndpoint.ListConversationItemsAsync("conversation-id", query); + +foreach (var item in items) +{ + Debug.Log(item.ToJsonString()); +} +``` + +#### [Create Conversation Item](https://platform.openai.com/docs/api-reference/conversations/create-item) + +Create a new conversation item for a conversation with the given ID. + +```csharp +var api = new OpenAIClient(); +var items = new List +{ + new Message(Role.User, "Hello!"), + new Message(Role.Assistant, "Hi! How can I help you?") +} +var addedItems = await api.ConversationsEndpoint.CreateConversationItemsAsync("conversation-id", items); + +foreach (var item in addedItems) +{ + Debug.Log(item.ToJsonString()); +} +``` + +#### [Retrieve Conversation Item](https://platform.openai.com/docs/api-reference/conversations/retrieve-item) + +Get a conversation item by id. + +```csharp +var api = new OpenAIClient(); +var item = await api.ConversationsEndpoint.GetConversationItemAsync("conversation-id", "item-id"); +Debug.Log(item.ToJsonString()); +``` + +#### [Delete Conversation Item](https://platform.openai.com/docs/api-reference/conversations/delete-item) + +Delete a conversation item by id. + +```csharp +var api = new OpenAIClient(); +var isDeleted = await api.ConversationsEndpoint.DeleteConversationItemAsync("conversation-id", "item-id"); +Assert.IsTrue(isDeleted); +``` + +--- + ### [Realtime](https://platform.openai.com/docs/api-reference/realtime) > [!WARNING] @@ -903,7 +1026,8 @@ Debug.Log($"Retrieve thread {thread.Id} -> {thread.CreatedAt}"); Modifies a thread. -> Note: Only the metadata can be modified. +> [!NOTE] +> Only the metadata can be modified. ```csharp var api = new OpenAIClient(); @@ -981,7 +1105,8 @@ Debug.Log($"{message.Id}: {message.Role}: {message.PrintContent()}"); Modify a message. -> Note: Only the message metadata can be modified. +> [!NOTE] +> Only the message metadata can be modified. ```csharp var api = new OpenAIClient(); @@ -1076,7 +1201,8 @@ Debug.Log($"[{run.Id}] {run.Status} | {run.CreatedAt}"); Modifies a run. -> Note: Only the metadata can be modified. +> [!NOTE] +> Only the metadata can be modified. ```csharp var api = new OpenAIClient(); @@ -1795,7 +1921,7 @@ Generates audio from the input text. ```csharp var api = new OpenAIClient(); var request = new SpeechRequest("Hello world!"); -var speechClip = await api.AudioEndpoint.CreateSpeechAsync(request); +var speechClip = await api.AudioEndpoint.GetSpeechAsync(request); audioSource.PlayOneShot(speechClip); Debug.Log(speechClip); ``` @@ -1807,7 +1933,7 @@ Generate streamed audio from the input text. ```csharp var api = new OpenAIClient(); var request = new SpeechRequest("Hello world!", responseFormat: SpeechResponseFormat.PCM); -var speechClip = await api.AudioEndpoint.CreateSpeechStreamAsync(request, partialClip => +var speechClip = await api.AudioEndpoint.GetSpeechAsync(request, partialClip => { audioSource.PlayOneShot(partialClip); }); diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Models/Model.cs b/OpenAI/Packages/com.openai.unity/Runtime/Models/Model.cs index 69544e5e..10fef867 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Models/Model.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Models/Model.cs @@ -225,7 +225,7 @@ internal Model( /// - Context Window: 400,000 context window
/// - Max Output Tokens: 128,000 max output tokens /// - public static Model GPT_5 { get; } = new("gpt-5", "openai"); + public static Model GPT5 { get; } = new("gpt-5", "openai"); /// /// GPT-5 mini is a faster, more cost-efficient version of GPT-5. It's great for well-defined tasks and precise prompts. @@ -234,7 +234,7 @@ internal Model( /// - Context Window: 400,000 context window
/// - Max Output Tokens: 128,000 max output tokens /// - public static Model GPT_5_Mini { get; } = new("gpt-5-mini", "openai"); + public static Model GPT5_Mini { get; } = new("gpt-5-mini", "openai"); /// /// GPT-5 Nano is our fastest, cheapest version of GPT-5. It's great for summarization and classification tasks. @@ -243,7 +243,7 @@ internal Model( /// - Context Window: 400,000 context window
/// - Max Output Tokens: 128,000 max output tokens /// - public static Model GPT_5_Nano { get; } = new("gpt-5-nano", "openai"); + public static Model GPT5_Nano { get; } = new("gpt-5-nano", "openai"); /// /// GPT-5 Chat points to the GPT-5 snapshot currently used in ChatGPT. @@ -254,7 +254,7 @@ internal Model( /// - Context Window: 128,000 context window
/// - Max Output Tokens: 16,384 max output tokens /// - public static Model GPT_5_Chat { get; } = new("gpt-5-chat-latest", "openai"); + public static Model GPT5_Chat { get; } = new("gpt-5-chat-latest", "openai"); /// /// ChatGPT-4o points to the GPT-4o snapshot currently used in ChatGPT. @@ -269,7 +269,7 @@ internal Model( public static Model ChatGPT4o { get; } = new("chatgpt-4o-latest", "openai"); /// - /// GPT-4o (“o” for “omni”) is our versatile, high-intelligence flagship model. + /// GPT-4o ('o' for 'omni') is our versatile, high-intelligence flagship model. /// It accepts both text and image inputs, and produces text outputs (including Structured Outputs). /// It is the best model for most tasks, and is our most capable model outside of our o-series models. /// @@ -280,7 +280,7 @@ internal Model( public static Model GPT4o { get; } = new("gpt-4o", "openai"); /// - /// GPT-4o mini (“o” for “omni”) is a fast, affordable small model for focused tasks. + /// GPT-4o mini ('o' for 'omni') is a fast, affordable small model for focused tasks. /// It accepts both text and image inputs, and produces text outputs (including Structured Outputs). /// It is ideal for fine-tuning, and model outputs from a larger model like GPT-4o can be distilled /// to GPT-4o-mini to produce similar results at lower cost and latency. @@ -512,14 +512,14 @@ internal Model( public static Model GPT_Image_1 { get; } = new("gpt-image-1", "openai"); /// - /// DALL·E is an AI system that creates realistic images and art from a natural language description. - /// DALL·E 3 currently supports the ability, given a prompt, to create a new image with a specific size. + /// DALL-E is an AI system that creates realistic images and art from a natural language description. + /// DALL-E 3 currently supports the ability, given a prompt, to create a new image with a specific size. /// public static Model DallE_3 { get; } = new("dall-e-3", "openai"); /// - /// DALL·E is an AI system that creates realistic images and art from a natural language description. - /// Older than DALL·E 3, DALL·E 2 offers more control in prompting and more requests at once. + /// DALL-E is an AI system that creates realistic images and art from a natural language description. + /// Older than DALL-E 3, DALL-E 2 offers more control in prompting and more requests at once. /// public static Model DallE_2 { get; } = new("dall-e-2", "openai"); @@ -535,7 +535,7 @@ internal Model( /// - Context Window: 400,000 tokens
/// - Max Output Tokens: 128,000 tokens /// - public static Model GPT_5_Codex { get; } = new("gpt-5-codex", "openai"); + public static Model GPT5_Codex { get; } = new("gpt-5-codex", "openai"); /// /// codex-mini-latest is a fine-tuned version of o4-mini specifically for use in Codex CLI. diff --git a/OpenAI/Packages/com.openai.unity/Samples~/Responses/ResponsesBehaviour.cs b/OpenAI/Packages/com.openai.unity/Samples~/Responses/ResponsesBehaviour.cs index 259d8363..c034c64a 100644 --- a/OpenAI/Packages/com.openai.unity/Samples~/Responses/ResponsesBehaviour.cs +++ b/OpenAI/Packages/com.openai.unity/Samples~/Responses/ResponsesBehaviour.cs @@ -134,7 +134,7 @@ private async void SubmitChat() try { - var request = new CreateResponseRequest(textInput: userInput, conversationId: conversation, tools: assistantTools, model: Model.GPT4_1_Nano); + var request = new CreateResponseRequest(textInput: userInput, conversationId: conversation, tools: assistantTools, model: Model.GPT5_Nano); async Task StreamEventHandler(string eventName, IServerSentEvent sseEvent) { switch (sseEvent) diff --git a/README.md b/README.md index 4c2d6584..e4492351 100644 --- a/README.md +++ b/README.md @@ -80,6 +80,15 @@ openupm add com.openai.unity - [List Input Items](#list-input-items) - [Cancel Response](#cancel-response) - [Delete Response](#delete-response) +- [Conversations](#conversations) :new: + - [Create Conversation](#create-conversation) :new: + - [Retrieve Conversation](#retrieve-conversation) :new: + - [Update Conversation](#update-conversation) :new: + - [Delete Conversation](#delete-conversation) :new: + - [List Conversation Items](#list-conversation-items) :new: + - [Create Conversation Item](#create-conversation-item) :new: + - [Retrieve Conversation Item](#retrieve-conversation-item) :new: + - [Delete Conversation Item](#delete-conversation-item) :new: - [Realtime](#realtime) - [Create Realtime Session](#create-realtime-session) - [Client Events](#client-events) @@ -452,7 +461,7 @@ Creates a model response. Provide text or image inputs to generate text or JSON var api = new OpenAIClient(); var response = await api.ResponsesEndpoint.CreateModelResponseAsync("Tell me a three sentence bedtime story about a unicorn."); var responseItem = response.Output.LastOrDefault(); -Debug.Log($"{messageItem.Role}:{textContent.Text}"); +Debug.Log($"{responseItem.Role}:{responseItem}"); response.PrintUsage(); ``` @@ -469,7 +478,7 @@ var tools = new List { Tool.GetOrCreateTool(typeof(DateTimeUtility), nameof(DateTimeUtility.GetDateTime)) }; -var request = new CreateResponseRequest(conversation, Model.GPT4_1_Nano, tools: tools); +var request = new CreateResponseRequest(conversation, Model.GPT5_Nano, tools: tools); async Task StreamCallback(string @event, IServerSentEvent sseEvent) { @@ -482,7 +491,7 @@ async Task StreamCallback(string @event, IServerSentEvent sseEvent) conversation.Add(functionToolCall); var output = await functionToolCall.InvokeFunctionAsync(); conversation.Add(output); - await api.ResponsesEndpoint.CreateModelResponseAsync(new(conversation, Model.GPT4_1_Nano, tools: tools, toolChoice: "none"), StreamCallback); + await api.ResponsesEndpoint.CreateModelResponseAsync(new(conversation, Model.GPT5_Nano, tools: tools, toolChoice: "none"), StreamCallback); break; } } @@ -541,6 +550,120 @@ Assert.IsTrue(isDeleted); --- +### [Conversations](https://platform.openai.com/docs/api-reference/conversations) + +Create and manage conversations to store and retrieve conversation state across Response API calls. + +The Conversations API is accessed via `OpenAIClient.ConversationsEndpoint` + +#### [Create Conversation](https://platform.openai.com/docs/api-reference/conversations/create) + +Create a conversation. + +```csharp +var api = new OpenAIClient(); +conversation = await api.ConversationsEndpoint.CreateConversationAsync( + new CreateConversationRequest(new Message(Role.Developer, systemPrompt))); +Debug.Log(conversation.ToString()); +// use the conversation object when creating responses. +var request = await api.ResponsesEndpoint.CreateResponseAsync( + new CreateResponseRequest(textInput: "Hello!", conversationId: conversation, model: Model.GPT5_Nano)); +var response = await openAI.ResponsesEndpoint.CreateModelResponseAsync(request); +var responseItem = response.Output.LastOrDefault(); +Debug.Log($"{responseItem.Role}:{responseItem}"); +response.PrintUsage(); +``` + +#### [Retrieve Conversation](https://platform.openai.com/docs/api-reference/conversations/retrieve) + +Get a conversation by id. + +```csharp +var api = new OpenAIClient(); +var conversation = await api.ConversationsEndpoint.GetConversationAsync("conversation-id"); +Debug.Log(conversation.ToString()); +``` + +#### [Update Conversation](https://platform.openai.com/docs/api-reference/conversations/update) + +Update a conversation with custom metadata. + +```csharp +var api = new OpenAIClient(); +var metadata = new Dictionary +{ + { "favorite_color", "blue" }, + { "favorite_food", "pizza" } +}; +var updatedConversation = await api.ConversationsEndpoint.UpdateConversationAsync("conversation-id", metadata); +``` + +#### [Delete Conversation](https://platform.openai.com/docs/api-reference/conversations/delete) + +Delete a conversation by id. + +```csharp +var api = new OpenAIClient(); +var isDeleted = await api.ConversationsEndpoint.DeleteConversationAsync("conversation-id"); +Assert.IsTrue(isDeleted); +``` + +#### [List Conversation Items](https://platform.openai.com/docs/api-reference/conversations/list-items) + +List all items for a conversation with the given ID. + +```csharp +var api = new OpenAIClient(); +var query = new ListQuery(limit: 10); +var items = await api.ConversationsEndpoint.ListConversationItemsAsync("conversation-id", query); + +foreach (var item in items) +{ + Debug.Log(item.ToJsonString()); +} +``` + +#### [Create Conversation Item](https://platform.openai.com/docs/api-reference/conversations/create-item) + +Create a new conversation item for a conversation with the given ID. + +```csharp +var api = new OpenAIClient(); +var items = new List +{ + new Message(Role.User, "Hello!"), + new Message(Role.Assistant, "Hi! How can I help you?") +} +var addedItems = await api.ConversationsEndpoint.CreateConversationItemsAsync("conversation-id", items); + +foreach (var item in addedItems) +{ + Debug.Log(item.ToJsonString()); +} +``` + +#### [Retrieve Conversation Item](https://platform.openai.com/docs/api-reference/conversations/retrieve-item) + +Get a conversation item by id. + +```csharp +var api = new OpenAIClient(); +var item = await api.ConversationsEndpoint.GetConversationItemAsync("conversation-id", "item-id"); +Debug.Log(item.ToJsonString()); +``` + +#### [Delete Conversation Item](https://platform.openai.com/docs/api-reference/conversations/delete-item) + +Delete a conversation item by id. + +```csharp +var api = new OpenAIClient(); +var isDeleted = await api.ConversationsEndpoint.DeleteConversationItemAsync("conversation-id", "item-id"); +Assert.IsTrue(isDeleted); +``` + +--- + ### [Realtime](https://platform.openai.com/docs/api-reference/realtime) > [!WARNING]