Google genai response multimodality support

Author: 楚孔响 <[email protected]>
Signed-off-by: 楚孔响 <[email protected]>
Signed-off-by: ckx521 <[email protected]>

Author: qq1014853731

models/spring-ai-google-genai/src/main/java/org/springframework/ai/google/genai/GoogleGenAiChatModel.java

@@ -91,6 +91,7 @@
 import org.springframework.util.Assert;
 import org.springframework.util.CollectionUtils;
 import org.springframework.util.StringUtils;
+import org.springframework.util.MimeType;
 
 /**
  * Google GenAI Chat Model implementation that provides access to Google's Gemini language
@@ -626,7 +627,19 @@ protected List<Generation> responseCandidateToGeneration(Candidate candidate) {
 				.parts()
 				.orElse(List.of())
 				.stream()
-				.map(part -> new AssistantMessage(part.text().orElse(""), messageMetadata))
+				.map(part -> {
+					// Multimodality Response Support
+					List<Media> media = part.inlineData()
+							.filter(blob -> blob.data().isPresent() && blob.mimeType().isPresent())
+							.map(blob -> Media
+									.builder()
+									.mimeType(MimeType.valueOf(blob.mimeType().get()))
+									.data(blob.data().get())
+									.build())
+							.map(List::of)
+							.orElse(List.of());
+					return new AssistantMessage(part.text().orElse(""), messageMetadata, List.of(), media);
+				})
 				.map(assistantMessage -> new Generation(assistantMessage, chatGenerationMetadata))
 				.toList();
 		}
@@ -725,6 +738,10 @@ GeminiRequest createGeminiRequest(Prompt prompt) {
 			configBuilder.systemInstruction(systemContents.get(0));
 		}
 
+		if (!CollectionUtils.isEmpty(requestOptions.getResponseModalities())) {
+			configBuilder.responseModalities(requestOptions.getResponseModalities());
+		}
+
 		GenerateContentConfig config = configBuilder.build();
 
 		// Create message contents
@@ -850,7 +867,7 @@ public static final class Builder {
 		private GoogleGenAiChatOptions defaultOptions = GoogleGenAiChatOptions.builder()
 			.temperature(0.7)
 			.topP(1.0)
-			.model(GoogleGenAiChatModel.ChatModel.GEMINI_2_0_FLASH)
+			.model(ChatModel.GEMINI_2_0_FLASH)
 			.build();
 
 		private ToolCallingManager toolCallingManager;

models/spring-ai-google-genai/src/main/java/org/springframework/ai/google/genai/GoogleGenAiChatOptions.java

@@ -113,6 +113,12 @@ public class GoogleGenAiChatOptions implements ToolCallingChatOptions {
 	 */
 	private @JsonProperty("thinkingBudget") Integer thinkingBudget;
 
+	/**
+	 * Optional. Response Modalities.
+	 * @see com.google.genai.types.Modality.Known
+	 */
+	private @JsonProperty("responseModalities") List<String> responseModalities = new ArrayList<>();
+
 	/**
 	 * Collection of {@link ToolCallback}s to be used for tool calling in the chat
 	 * completion requests.
@@ -174,6 +180,7 @@ public static GoogleGenAiChatOptions fromOptions(GoogleGenAiChatOptions fromOpti
 		options.setToolContext(fromOptions.getToolContext());
 		options.setThinkingBudget(fromOptions.getThinkingBudget());
 		options.setLabels(fromOptions.getLabels());
+		options.setResponseModalities(fromOptions.getResponseModalities());
 		return options;
 	}
 
@@ -355,6 +362,15 @@ public void setToolContext(Map<String, Object> toolContext) {
 		this.toolContext = toolContext;
 	}
 
+	public List<String> getResponseModalities() {
+		return responseModalities;
+	}
+
+	public void setResponseModalities(List<String> responseModalities) {
+		Assert.notNull(responseModalities, "responseModalities cannot be null");
+		this.responseModalities = responseModalities;
+	}
+
 	@Override
 	public boolean equals(Object o) {
 		if (this == o) {
@@ -376,15 +392,17 @@ public boolean equals(Object o) {
 				&& Objects.equals(this.toolNames, that.toolNames)
 				&& Objects.equals(this.safetySettings, that.safetySettings)
 				&& Objects.equals(this.internalToolExecutionEnabled, that.internalToolExecutionEnabled)
-				&& Objects.equals(this.toolContext, that.toolContext) && Objects.equals(this.labels, that.labels);
+				&& Objects.equals(this.toolContext, that.toolContext) && Objects.equals(this.labels, that.labels)
+				&& Objects.equals(this.responseModalities, that.responseModalities);
 	}
 
 	@Override
 	public int hashCode() {
 		return Objects.hash(this.stopSequences, this.temperature, this.topP, this.topK, this.candidateCount,
 				this.frequencyPenalty, this.presencePenalty, this.thinkingBudget, this.maxOutputTokens, this.model,
 				this.responseMimeType, this.toolCallbacks, this.toolNames, this.googleSearchRetrieval,
-				this.safetySettings, this.internalToolExecutionEnabled, this.toolContext, this.labels);
+				this.safetySettings, this.internalToolExecutionEnabled, this.toolContext, this.labels,
+				this.responseModalities);
 	}
 
 	@Override
@@ -396,7 +414,7 @@ public String toString() {
 				+ this.model + '\'' + ", responseMimeType='" + this.responseMimeType + '\'' + ", toolCallbacks="
 				+ this.toolCallbacks + ", toolNames=" + this.toolNames + ", googleSearchRetrieval="
 				+ this.googleSearchRetrieval + ", safetySettings=" + this.safetySettings + ", labels=" + this.labels
-				+ '}';
+				+ ", responseModalities=" + this.responseModalities + '}';
 	}
 
 	@Override
@@ -530,6 +548,18 @@ public Builder labels(Map<String, String> labels) {
 			return this;
 		}
 
+		public Builder responseModalities(List<String> responseModalities) {
+			Assert.notNull(responseModalities, "responseModalities must not be null");
+			this.options.responseModalities = responseModalities;
+			return this;
+		}
+
+		public Builder responseModalitie(String responseModalitie) {
+			Assert.hasText(responseModalitie, "responseModalitie must not be empty");
+			this.options.responseModalities.add(responseModalitie);
+			return this;
+		}
+
 		public GoogleGenAiChatOptions build() {
 			return this.options;
 		}

models/spring-ai-google-genai/src/test/java/org/springframework/ai/google/genai/GoogleGenAiChatOptionsTest.java

@@ -16,8 +16,10 @@
 
 package org.springframework.ai.google.genai;
 
+import java.util.List;
 import java.util.Map;
 
+import com.google.genai.types.Modality;
 import org.junit.jupiter.api.Test;
 
 import static org.assertj.core.api.Assertions.assertThat;
@@ -153,4 +155,13 @@ public void testToStringWithLabels() {
 		assertThat(toString).contains("test-model");
 	}
 
+	@Test
+	public void testResponseMultimodality() {
+		GoogleGenAiChatOptions options = GoogleGenAiChatOptions.builder()
+				.responseModalities(List.of(Modality.Known.TEXT.name(), Modality.Known.IMAGE.name()))
+				.build();
+		String toString = options.toString();
+		assertThat(toString).contains("responseModalities=[TEXT, IMAGE]");
+	}
+
 }