Skip to content

chore: support the new genai endpoint format #297

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

chore: support the new genai endpoint format #297

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 16 additions & 3 deletions java-cfenv-boot/src/main/java/io/pivotal/cfenv/spring/boot/GenAIChatCfEnvProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,13 +28,18 @@
*
* @author Stuart Charlton
* @author Ed King
* @author Gareth Evans
**/
public class GenAIChatCfEnvProcessor implements CfEnvProcessor {

@Override
public boolean accept(CfService service) {
boolean isGenAIService = service.existsByTagIgnoreCase("genai") || service.existsByLabelStartsWith("genai");
if (isGenAIService) {
boolean endpointFormat = service.getCredentials().getMap().containsKey("endpoint");
if (endpointFormat) {
return true;
}
ArrayList<String> modelCapabilities = (ArrayList<String>) service.getCredentials().getMap().get("model_capabilities");
return modelCapabilities.contains("chat");
}
Expand All @@ -46,9 +51,17 @@ public boolean accept(CfService service) {
public void process(CfCredentials cfCredentials, Map<String, Object> properties) {
properties.put("spring.ai.openai.api-key", "redundant");

properties.put("spring.ai.openai.chat.base-url", cfCredentials.getString("api_base"));
properties.put("spring.ai.openai.chat.api-key", cfCredentials.getString("api_key"));
properties.put("spring.ai.openai.chat.options.model", cfCredentials.getString("model_name"));
boolean endpointFormat = cfCredentials.getMap().containsKey("endpoint");
if (endpointFormat) {
Map<String, String> endpoint = (Map<String, String>)cfCredentials.getMap().get("endpoint");
properties.put("spring.ai.openai.chat.base-url", endpoint.get("api_base") + "/openai");
properties.put("spring.ai.openai.chat.api-key", endpoint.get("api_key"));
properties.put("spring.ai.openai.chat.options.model", "tanzu://chat");
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If I understand correctly, this logic will always set the spring.ai.openai.chat.* properties for all bindings that contain the endpoint key, even if there is no chat-capable model behind the endpoint? Is that a problem? I guess that if an app were to make a chat request, it would receive some sort of 404 model not found response, which maybe is fine?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I guess this would only be an issue if the app / spring ai attempts to test connectivity to all configured models at startup 🤔

} else {
properties.put("spring.ai.openai.chat.base-url", cfCredentials.getString("api_base"));
properties.put("spring.ai.openai.chat.api-key", cfCredentials.getString("api_key"));
properties.put("spring.ai.openai.chat.options.model", cfCredentials.getString("model_name"));
}
}

@Override
Expand Down
19 changes: 16 additions & 3 deletions java-cfenv-boot/src/main/java/io/pivotal/cfenv/spring/boot/GenAIEmbeddingCfEnvProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,13 +28,18 @@
*
* @author Stuart Charlton
* @author Ed King
* @author Gareth Evans
**/
public class GenAIEmbeddingCfEnvProcessor implements CfEnvProcessor {

@Override
public boolean accept(CfService service) {
boolean isGenAIService = service.existsByTagIgnoreCase("genai") || service.existsByLabelStartsWith("genai");
if (isGenAIService) {
boolean endpointFormat = service.getCredentials().getMap().containsKey("endpoint");
if (endpointFormat) {
return true;
}
ArrayList<String> modelCapabilities = (ArrayList<String>) service.getCredentials().getMap().get("model_capabilities");
return modelCapabilities.contains("embedding");
}
Expand All @@ -46,9 +51,17 @@ public boolean accept(CfService service) {
public void process(CfCredentials cfCredentials, Map<String, Object> properties) {
properties.put("spring.ai.openai.api-key", "redundant");

properties.put("spring.ai.openai.embedding.base-url", cfCredentials.getString("api_base"));
properties.put("spring.ai.openai.embedding.api-key", cfCredentials.getString("api_key"));
properties.put("spring.ai.openai.embedding.options.model", cfCredentials.getString("model_name"));
boolean endpointFormat = cfCredentials.getMap().containsKey("endpoint");
if (endpointFormat) {
Map<String, String> endpoint = (Map<String, String>)cfCredentials.getMap().get("endpoint");
properties.put("spring.ai.openai.embedding.base-url", endpoint.get("api_base") + "/openai");
properties.put("spring.ai.openai.embedding.api-key", endpoint.get("api_key"));
properties.put("spring.ai.openai.embedding.options.model", "tanzu://embedding");
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same note as above, this will always try to configure an embedding model, even when there is none.

} else {
properties.put("spring.ai.openai.embedding.base-url", cfCredentials.getString("api_base"));
properties.put("spring.ai.openai.embedding.api-key", cfCredentials.getString("api_key"));
properties.put("spring.ai.openai.embedding.options.model", cfCredentials.getString("model_name"));
}
}

@Override
Expand Down
23 changes: 23 additions & 0 deletions java-cfenv-boot/src/test/java/io/pivotal/cfenv/spring/boot/GenAIChatCfEnvProcessorTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
/**
* @author Stuart Charlton
* @author Ed King
* @author Gareth Evans
*/
public class GenAIChatCfEnvProcessorTests extends AbstractCfEnvTests {

Expand All @@ -48,4 +49,26 @@ public void testGenAIBootPropertiesWithChatModelCapability() {
assertThat(getEnvironment().getProperty("spring.ai.openai.audio.transcription.options.model")).isNull();
assertThat(getEnvironment().getProperty("spring.ai.openai.audio.speech.options.model")).isNull();
}

@Test
public void testGenAIBootPropertiesWithChatModelCapabilityEndpointFormat() {
String TEST_GENAI_JSON_FILE = "test-genai-endpoint-chat-model.json";

String EXPECTED_URI_FROM_JSON_FILE = "https://genai-proxy.tpcf.io/test/openai";
String EXPECTED_TOKEN_FROM_JSON_FILE = "sk-KW5kiNOKDd_1dFxsAjpVa";
String EXPECTED_MODEL_FROM_JSON_FILE = "tanzu://chat";

mockVcapServices(getServicesPayload(readTestDataFile(TEST_GENAI_JSON_FILE)));

assertThat(getEnvironment().getProperty("spring.ai.openai.api-key")).isEqualTo("redundant");

assertThat(getEnvironment().getProperty("spring.ai.openai.chat.base-url")).isEqualTo(EXPECTED_URI_FROM_JSON_FILE);
assertThat(getEnvironment().getProperty("spring.ai.openai.chat.api-key")).isEqualTo(EXPECTED_TOKEN_FROM_JSON_FILE);
assertThat(getEnvironment().getProperty("spring.ai.openai.chat.options.model")).isEqualTo(EXPECTED_MODEL_FROM_JSON_FILE);

assertThat(getEnvironment().getProperty("spring.ai.openai.embedding.options.model")).isNotNull();
assertThat(getEnvironment().getProperty("spring.ai.openai.image.options.model")).isNull();
assertThat(getEnvironment().getProperty("spring.ai.openai.audio.transcription.options.model")).isNull();
assertThat(getEnvironment().getProperty("spring.ai.openai.audio.speech.options.model")).isNull();
}
}
23 changes: 23 additions & 0 deletions ...nv-boot/src/test/java/io/pivotal/cfenv/spring/boot/GenAIEmbeddingCfEnvProcessorTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
/**
* @author Stuart Charlton
* @author Ed King
* @author Gareth Evans
*/
public class GenAIEmbeddingCfEnvProcessorTests extends AbstractCfEnvTests {

Expand All @@ -48,4 +49,26 @@ public void testGenAIBootPropertiesWithEmbeddingModelCapability() {
assertThat(getEnvironment().getProperty("spring.ai.openai.audio.transcription.options.model")).isNull();
assertThat(getEnvironment().getProperty("spring.ai.openai.audio.speech.options.model")).isNull();
}

@Test
public void testGenAIBootPropertiesWithEmbeddingModelCapabilityEndpointFormat() {
String TEST_GENAI_JSON_FILE = "test-genai-endpoint-embedding-model.json";

String EXPECTED_URI_FROM_JSON_FILE = "https://genai-proxy.tpcf.io/test/openai";
String EXPECTED_TOKEN_FROM_JSON_FILE = "sk-KW5kiNOKDd_1dFxsAjpVa";
String EXPECTED_MODEL_FROM_JSON_FILE = "tanzu://embedding";

mockVcapServices(getServicesPayload(readTestDataFile(TEST_GENAI_JSON_FILE)));

assertThat(getEnvironment().getProperty("spring.ai.openai.api-key")).isEqualTo("redundant");

assertThat(getEnvironment().getProperty("spring.ai.openai.embedding.base-url")).isEqualTo(EXPECTED_URI_FROM_JSON_FILE);
assertThat(getEnvironment().getProperty("spring.ai.openai.embedding.api-key")).isEqualTo(EXPECTED_TOKEN_FROM_JSON_FILE);
assertThat(getEnvironment().getProperty("spring.ai.openai.embedding.options.model")).isEqualTo(EXPECTED_MODEL_FROM_JSON_FILE);

assertThat(getEnvironment().getProperty("spring.ai.openai.chat.options.model")).isNotNull();
assertThat(getEnvironment().getProperty("spring.ai.openai.image.options.model")).isNull();
assertThat(getEnvironment().getProperty("spring.ai.openai.audio.transcription.options.model")).isNull();
assertThat(getEnvironment().getProperty("spring.ai.openai.audio.speech.options.model")).isNull();
}
}
20 changes: 20 additions & 0 deletions ...-boot/src/test/resources/io/pivotal/cfenv/spring/boot/test-genai-endpoint-chat-model.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
{
"credentials": {
"endpoint": {
"api_base": "https://genai-proxy.tpcf.io/test",
"api_key": "sk-KW5kiNOKDd_1dFxsAjpVa",
"config_url": "https://genai-proxy.tpcf.io/test/config/v1/endpoint"
}
},
"instance_name": "genai",
"label": "genai",
"name": "genai",
"plan": "meta-llama/Meta-Llama-3-8B",
"provider": null,
"syslog_drain_url": null,
"tags": [
"genai",
"llm"
],
"volume_mounts": []
}
20 changes: 20 additions & 0 deletions .../src/test/resources/io/pivotal/cfenv/spring/boot/test-genai-endpoint-embedding-model.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
{
"credentials": {
"endpoint": {
"api_base": "https://genai-proxy.tpcf.io/test",
"api_key": "sk-KW5kiNOKDd_1dFxsAjpVa",
"config_url": "https://genai-proxy.tpcf.io/test/config/v1/endpoint"
}
},
"instance_name": "genai",
"label": "genai",
"name": "genai",
"plan": "mixedbread-ai/mxbai-embed-large-v1",
"provider": null,
"syslog_drain_url": null,
"tags": [
"genai",
"llm"
],
"volume_mounts": []
}