Google genai response multimodality support

models/spring-ai-google-genai/src/main/java/org/springframework/ai/google/genai/GoogleGenAiChatModel.java

@@ -90,6 +90,7 @@
 import org.springframework.retry.support.RetryTemplate;
 import org.springframework.util.Assert;
 import org.springframework.util.CollectionUtils;
+import org.springframework.util.MimeType;
 import org.springframework.util.StringUtils;
 
 /**
@@ -621,14 +622,18 @@ protected List<Generation> responseCandidateToGeneration(Candidate candidate) {
 			return List.of(new Generation(assistantMessage, chatGenerationMetadata));
 		}
 		else {
-			return candidate.content()
-				.get()
-				.parts()
-				.orElse(List.of())
-				.stream()
-				.map(part -> new AssistantMessage(part.text().orElse(""), messageMetadata))
-				.map(assistantMessage -> new Generation(assistantMessage, chatGenerationMetadata))
-				.toList();
+			return candidate.content().flatMap(Content::parts).orElse(List.of()).stream().map(part -> {
+				// Multimodality Response Support
+				List<Media> media = part.inlineData()
+					.filter(blob -> blob.data().isPresent() && blob.mimeType().isPresent())
+					.map(blob -> Media.builder()
+						.mimeType(MimeType.valueOf(blob.mimeType().get()))
+						.data(blob.data().get())
+						.build())
+					.map(List::of)
+					.orElse(List.of());
+				return new AssistantMessage(part.text().orElse(""), messageMetadata, List.of(), media);
+			}).map(assistantMessage -> new Generation(assistantMessage, chatGenerationMetadata)).toList();
 		}
 	}
 
@@ -725,6 +730,10 @@ GeminiRequest createGeminiRequest(Prompt prompt) {
 			configBuilder.systemInstruction(systemContents.get(0));
 		}
 
+		if (!CollectionUtils.isEmpty(requestOptions.getResponseModalities())) {
+			configBuilder.responseModalities(requestOptions.getResponseModalities());
+		}
+
 		GenerateContentConfig config = configBuilder.build();
 
 		// Create message contents
@@ -850,7 +859,7 @@ public static final class Builder {
 		private GoogleGenAiChatOptions defaultOptions = GoogleGenAiChatOptions.builder()
 			.temperature(0.7)
 			.topP(1.0)
-			.model(GoogleGenAiChatModel.ChatModel.GEMINI_2_0_FLASH)
+			.model(ChatModel.GEMINI_2_0_FLASH)
 			.build();
 
 		private ToolCallingManager toolCallingManager;

models/spring-ai-google-genai/src/main/java/org/springframework/ai/google/genai/GoogleGenAiChatOptions.java

@@ -113,6 +113,12 @@ public class GoogleGenAiChatOptions implements ToolCallingChatOptions {
 	 */
 	private @JsonProperty("thinkingBudget") Integer thinkingBudget;
 
+	/**
+	 * Optional. Response Modalities.
+	 * @see com.google.genai.types.Modality.Known
+	 */
+	private @JsonProperty("responseModalities") List<String> responseModalities = new ArrayList<>();
+
 	/**
 	 * Collection of {@link ToolCallback}s to be used for tool calling in the chat
 	 * completion requests.
@@ -174,6 +180,7 @@ public static GoogleGenAiChatOptions fromOptions(GoogleGenAiChatOptions fromOpti
 		options.setToolContext(fromOptions.getToolContext());
 		options.setThinkingBudget(fromOptions.getThinkingBudget());
 		options.setLabels(fromOptions.getLabels());
+		options.setResponseModalities(fromOptions.getResponseModalities());
 		return options;
 	}
 
@@ -355,6 +362,15 @@ public void setToolContext(Map<String, Object> toolContext) {
 		this.toolContext = toolContext;
 	}
 
+	public List<String> getResponseModalities() {
+		return this.responseModalities;
+	}
+
+	public void setResponseModalities(List<String> responseModalities) {
+		Assert.notNull(responseModalities, "responseModalities cannot be null");
+		this.responseModalities = responseModalities;
+	}
+
 	@Override
 	public boolean equals(Object o) {
 		if (this == o) {
@@ -376,15 +392,17 @@ public boolean equals(Object o) {
 				&& Objects.equals(this.toolNames, that.toolNames)
 				&& Objects.equals(this.safetySettings, that.safetySettings)
 				&& Objects.equals(this.internalToolExecutionEnabled, that.internalToolExecutionEnabled)
-				&& Objects.equals(this.toolContext, that.toolContext) && Objects.equals(this.labels, that.labels);
+				&& Objects.equals(this.toolContext, that.toolContext) && Objects.equals(this.labels, that.labels)
+				&& Objects.equals(this.responseModalities, that.responseModalities);
 	}
 
 	@Override
 	public int hashCode() {
 		return Objects.hash(this.stopSequences, this.temperature, this.topP, this.topK, this.candidateCount,
 				this.frequencyPenalty, this.presencePenalty, this.thinkingBudget, this.maxOutputTokens, this.model,
 				this.responseMimeType, this.toolCallbacks, this.toolNames, this.googleSearchRetrieval,
-				this.safetySettings, this.internalToolExecutionEnabled, this.toolContext, this.labels);
+				this.safetySettings, this.internalToolExecutionEnabled, this.toolContext, this.labels,
+				this.responseModalities);
 	}
 
 	@Override
@@ -396,7 +414,7 @@ public String toString() {
 				+ this.model + '\'' + ", responseMimeType='" + this.responseMimeType + '\'' + ", toolCallbacks="
 				+ this.toolCallbacks + ", toolNames=" + this.toolNames + ", googleSearchRetrieval="
 				+ this.googleSearchRetrieval + ", safetySettings=" + this.safetySettings + ", labels=" + this.labels
-				+ '}';
+				+ ", responseModalities=" + this.responseModalities + '}';
 	}
 
 	@Override
@@ -530,6 +548,18 @@ public Builder labels(Map<String, String> labels) {
 			return this;
 		}
 
+		public Builder responseModalities(List<String> responseModalities) {
+			Assert.notNull(responseModalities, "responseModalities must not be null");
+			this.options.responseModalities = responseModalities;
+			return this;
+		}
+
+		public Builder responseModalitie(String responseModalitie) {
+			Assert.hasText(responseModalitie, "responseModalitie must not be empty");
+			this.options.responseModalities.add(responseModalitie);
+			return this;
+		}
+
 		public GoogleGenAiChatOptions build() {
 			return this.options;
 		}

models/spring-ai-google-genai/src/test/java/org/springframework/ai/google/genai/GoogleGenAiChatOptionsTest.java

@@ -16,8 +16,10 @@
 
 package org.springframework.ai.google.genai;
 
+import java.util.List;
 import java.util.Map;
 
+import com.google.genai.types.Modality;
 import org.junit.jupiter.api.Test;
 
 import static org.assertj.core.api.Assertions.assertThat;
@@ -153,4 +155,13 @@ public void testToStringWithLabels() {
 		assertThat(toString).contains("test-model");
 	}
 
+	@Test
+	public void testResponseMultimodality() {
+		GoogleGenAiChatOptions options = GoogleGenAiChatOptions.builder()
+			.responseModalities(List.of(Modality.Known.TEXT.name(), Modality.Known.IMAGE.name()))
+			.build();
+		String toString = options.toString();
+		assertThat(toString).contains("responseModalities=[TEXT, IMAGE]");
+	}
+
 }

pom.xml

@@ -282,7 +282,7 @@
 		<onnxruntime.version>1.19.2</onnxruntime.version>
 		<oci-sdk-version>3.63.1</oci-sdk-version>
 		<com.google.cloud.version>26.60.0</com.google.cloud.version>
-		<com.google.genai.version>1.10.0</com.google.genai.version>
+		<com.google.genai.version>1.15.0</com.google.genai.version>
 		<ibm.sdk.version>9.20.0</ibm.sdk.version>
 		<jsonschema.version>4.37.0</jsonschema.version>
 		<swagger-annotations.version>2.2.30</swagger-annotations.version>

spring-ai-model/src/main/java/org/springframework/ai/chat/model/MessageAggregator.java

@@ -34,6 +34,7 @@
 import org.springframework.ai.chat.metadata.PromptMetadata;
 import org.springframework.ai.chat.metadata.RateLimit;
 import org.springframework.ai.chat.metadata.Usage;
+import org.springframework.ai.content.Media;
 import org.springframework.util.CollectionUtils;
 import org.springframework.util.StringUtils;
 
@@ -60,6 +61,7 @@ public Flux<ChatResponse> aggregate(Flux<ChatResponse> fluxChatResponse,
 		AtomicReference<StringBuilder> messageTextContentRef = new AtomicReference<>(new StringBuilder());
 		AtomicReference<Map<String, Object>> messageMetadataMapRef = new AtomicReference<>();
 		AtomicReference<List<ToolCall>> toolCallsRef = new AtomicReference<>(new ArrayList<>());
+		AtomicReference<List<Media>> mediasRef = new AtomicReference<>(new ArrayList<>());
 
 		// ChatGeneration Metadata
 		AtomicReference<ChatGenerationMetadata> generationMetadataRef = new AtomicReference<>(
@@ -80,6 +82,7 @@ public Flux<ChatResponse> aggregate(Flux<ChatResponse> fluxChatResponse,
 			messageTextContentRef.set(new StringBuilder());
 			messageMetadataMapRef.set(new HashMap<>());
 			toolCallsRef.set(new ArrayList<>());
+			mediasRef.set(new ArrayList<>());
 			metadataIdRef.set("");
 			metadataModelRef.set("");
 			metadataUsagePromptTokensRef.set(0);
@@ -105,7 +108,9 @@ public Flux<ChatResponse> aggregate(Flux<ChatResponse> fluxChatResponse,
 				if (!CollectionUtils.isEmpty(outputMessage.getToolCalls())) {
 					toolCallsRef.get().addAll(outputMessage.getToolCalls());
 				}
-
+				if (!CollectionUtils.isEmpty(outputMessage.getMedia())) {
+					mediasRef.get().addAll(outputMessage.getMedia());
+				}
 			}
 			if (chatResponse.getMetadata() != null) {
 				if (chatResponse.getMetadata().getUsage() != null) {
@@ -137,6 +142,12 @@ public Flux<ChatResponse> aggregate(Flux<ChatResponse> fluxChatResponse,
 					List<ToolCall> toolCallsList = (List<ToolCall>) toolCallsFromMetadata;
 					toolCallsRef.get().addAll(toolCallsList);
 				}
+				Object mediasFromMetadata = chatResponse.getMetadata().get("medias");
+				if (mediasFromMetadata instanceof List) {
+					@SuppressWarnings("unchecked")
+					List<Media> mediasList = (List<Media>) mediasFromMetadata;
+					mediasRef.get().addAll(mediasList);
+				}
 
 			}
 		}).doOnComplete(() -> {
@@ -152,25 +163,20 @@ public Flux<ChatResponse> aggregate(Flux<ChatResponse> fluxChatResponse,
 				.promptMetadata(metadataPromptMetadataRef.get())
 				.build();
 
-			AssistantMessage finalAssistantMessage;
 			List<ToolCall> collectedToolCalls = toolCallsRef.get();
+			List<Media> collectedMedias = mediasRef.get();
 
-			if (!CollectionUtils.isEmpty(collectedToolCalls)) {
+			AssistantMessage finalAssistantMessage = new AssistantMessage(messageTextContentRef.get().toString(),
+					messageMetadataMapRef.get(), collectedToolCalls, collectedMedias);
 
-				finalAssistantMessage = new AssistantMessage(messageTextContentRef.get().toString(),
-						messageMetadataMapRef.get(), collectedToolCalls);
-			}
-			else {
-				finalAssistantMessage = new AssistantMessage(messageTextContentRef.get().toString(),
-						messageMetadataMapRef.get());
-			}
 			onAggregationComplete.accept(new ChatResponse(List.of(new Generation(finalAssistantMessage,
 
 					generationMetadataRef.get())), chatResponseMetadata));
 
 			messageTextContentRef.set(new StringBuilder());
 			messageMetadataMapRef.set(new HashMap<>());
 			toolCallsRef.set(new ArrayList<>());
+			mediasRef.set(new ArrayList<>());
 			metadataIdRef.set("");
 			metadataModelRef.set("");
 			metadataUsagePromptTokensRef.set(0);