codebase/testing-llm-responses-using-spring-ai-evaluators [BAEL-9143] (#18248)

* add codebase

* remove QA advisor for evaluator ChatClient

* add test case to check wrong answer relevant to topic via RelevancyEvaluator

* fix: grammar in test class

* delete unusable TestApplication

* remove unnecessary documents

Author: hardikSinghBehl

spring-ai-2/pom.xml

@@ -43,6 +43,10 @@
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-web</artifactId>
        </dependency>
+       <dependency>
+           <groupId>org.springframework.ai</groupId>
+           <artifactId>spring-ai-markdown-document-reader</artifactId>
+       </dependency>
        <dependency>
            <groupId>org.springframework.ai</groupId>
            <artifactId>spring-ai-ollama-spring-boot-starter</artifactId>

spring-ai-2/src/main/java/com/baeldung/springai/evaluator/Application.java

@@ -0,0 +1,29 @@
+package com.baeldung.springai.evaluator;
+
+import org.springframework.ai.autoconfigure.anthropic.AnthropicAutoConfiguration;
+import org.springframework.ai.autoconfigure.bedrock.converse.BedrockConverseProxyChatAutoConfiguration;
+import org.springframework.ai.autoconfigure.openai.OpenAiAutoConfiguration;
+import org.springframework.ai.autoconfigure.vectorstore.chroma.ChromaVectorStoreAutoConfiguration;
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.context.annotation.PropertySource;
+
+/**
+ * Excluding the below auto-configurations to avoid start up
+ * failure. Their corresponding starters are present on the classpath but are
+ * only needed by other articles in the shared codebase.
+ */
+@SpringBootApplication(exclude = {
+    OpenAiAutoConfiguration.class,
+    AnthropicAutoConfiguration.class,
+    ChromaVectorStoreAutoConfiguration.class,
+    BedrockConverseProxyChatAutoConfiguration.class
+})
+@PropertySource("classpath:application-evaluator.properties")
+public class Application {
+
+    public static void main(String[] args) {
+        SpringApplication.run(Application.class, args);
+    }
+
+}

spring-ai-2/src/main/java/com/baeldung/springai/evaluator/LLMConfiguration.java

@@ -0,0 +1,65 @@
+package com.baeldung.springai.evaluator;
+
+import org.springframework.ai.chat.client.ChatClient;
+import org.springframework.ai.chat.client.advisor.QuestionAnswerAdvisor;
+import org.springframework.ai.chat.model.ChatModel;
+import org.springframework.ai.embedding.EmbeddingModel;
+import org.springframework.ai.evaluation.FactCheckingEvaluator;
+import org.springframework.ai.evaluation.RelevancyEvaluator;
+import org.springframework.ai.ollama.OllamaChatModel;
+import org.springframework.ai.ollama.api.OllamaApi;
+import org.springframework.ai.ollama.api.OllamaOptions;
+import org.springframework.ai.ollama.management.ModelManagementOptions;
+import org.springframework.ai.ollama.management.PullModelStrategy;
+import org.springframework.ai.vectorstore.SimpleVectorStore;
+import org.springframework.ai.vectorstore.VectorStore;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+public class LLMConfiguration {
+
+    @Bean
+    public VectorStore vectorStore(EmbeddingModel embeddingModel) {
+        return SimpleVectorStore
+            .builder(embeddingModel)
+            .build();
+    }
+
+    @Bean
+    public ChatClient contentGenerator(ChatModel chatModel, VectorStore vectorStore) {
+        return ChatClient.builder(chatModel)
+            .defaultAdvisors(new QuestionAnswerAdvisor(vectorStore))
+            .build();
+    }
+
+    @Bean
+    public ChatClient contentEvaluator(
+        OllamaApi olamaApi,
+        @Value("${com.baeldung.evaluation.model}") String evaluationModel) {
+        ChatModel chatModel = OllamaChatModel.builder()
+            .ollamaApi(olamaApi)
+            .defaultOptions(OllamaOptions.builder()
+                .model(evaluationModel)
+                .build())
+            .modelManagementOptions(ModelManagementOptions.builder()
+                .pullModelStrategy(PullModelStrategy.WHEN_MISSING)
+                .build())
+            .build();
+        return ChatClient.builder(chatModel)
+            .build();
+    }
+
+    @Bean
+    public FactCheckingEvaluator factCheckingEvaluator(@Qualifier("contentEvaluator") ChatClient chatClient) {
+        return new FactCheckingEvaluator(chatClient.mutate());
+    }
+
+    @Bean
+    public RelevancyEvaluator relevancyEvaluator(@Qualifier("contentEvaluator") ChatClient chatClient) {
+        return new RelevancyEvaluator(chatClient.mutate());
+    }
+
+}

spring-ai-2/src/main/java/com/baeldung/springai/evaluator/VectorStoreInitializer.java

@@ -0,0 +1,41 @@
+package com.baeldung.springai.evaluator;
+
+import org.springframework.ai.document.Document;
+import org.springframework.ai.reader.markdown.MarkdownDocumentReader;
+import org.springframework.ai.reader.markdown.config.MarkdownDocumentReaderConfig;
+import org.springframework.ai.transformer.splitter.TokenTextSplitter;
+import org.springframework.ai.vectorstore.VectorStore;
+import org.springframework.boot.ApplicationArguments;
+import org.springframework.boot.ApplicationRunner;
+import org.springframework.core.io.Resource;
+import org.springframework.core.io.support.ResourcePatternResolver;
+import org.springframework.stereotype.Component;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+@Component
+class VectorStoreInitializer implements ApplicationRunner {
+
+    private final VectorStore vectorStore;
+    private final ResourcePatternResolver resourcePatternResolver;
+
+    public VectorStoreInitializer(VectorStore vectorStore, ResourcePatternResolver resourcePatternResolver) {
+        this.vectorStore = vectorStore;
+        this.resourcePatternResolver = resourcePatternResolver;
+    }
+
+    @Override
+    public void run(ApplicationArguments args) throws IOException {
+        List<Document> documents = new ArrayList<>();
+        Resource[] resources = resourcePatternResolver.getResources("classpath:documents/*.md");
+        Arrays.stream(resources).forEach(resource -> {
+            MarkdownDocumentReader markdownDocumentReader = new MarkdownDocumentReader(resource, MarkdownDocumentReaderConfig.defaultConfig());
+            documents.addAll(markdownDocumentReader.read());
+        });
+        vectorStore.add(new TokenTextSplitter().split(documents));
+    }
+
+}

spring-ai-2/src/main/resources/application-evaluator.properties

@@ -0,0 +1,5 @@
+com.baeldung.evaluation.model=bespoke-minicheck
+
+spring.ai.ollama.chat.options.model=llama3.3
+spring.ai.ollama.embedding.options.model=nomic-embed-text
+spring.ai.ollama.init.pull-model-strategy=when_missing

spring-ai-2/src/main/resources/documents/leave-policy.md

@@ -0,0 +1,17 @@
+# Leave Management Policy
+
+We offer comprehensive leave benefits to support work-life balance.
+
+## Types of Leave
+- Annual Leave: 20 days per year, accrued monthly
+- Sick Leave: 14 days per year, requires medical certificate for 3+ consecutive days
+- Parental Leave: 16 weeks for primary caregivers, 4 weeks for secondary
+- Bereavement Leave: 5 days for immediate family
+
+## Application Process
+1. Submit request through HR portal minimum 2 weeks in advance (except sick leave)
+2. Manager reviews within 48 hours
+3. HR processes approved requests within 24 hours
+
+## Leave Balance
+Leave balance resets annually on January 1st. Maximum 5 days carry-forward allowed.

spring-ai-2/src/test/java/com/baeldung/springai/evaluator/LLMResponseEvaluatorLiveTest.java

@@ -0,0 +1,113 @@
+package com.baeldung.springai.evaluator;
+
+import org.junit.jupiter.api.Test;
+import org.springframework.ai.chat.client.ChatClient;
+import org.springframework.ai.chat.client.advisor.QuestionAnswerAdvisor;
+import org.springframework.ai.chat.model.ChatResponse;
+import org.springframework.ai.document.Document;
+import org.springframework.ai.evaluation.EvaluationRequest;
+import org.springframework.ai.evaluation.EvaluationResponse;
+import org.springframework.ai.evaluation.FactCheckingEvaluator;
+import org.springframework.ai.evaluation.RelevancyEvaluator;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.context.annotation.Import;
+
+import java.util.List;
+
+import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
+
+@SpringBootTest
+@Import(TestcontainersConfiguration.class)
+class LLMResponseEvaluatorLiveTest {
+
+    @Autowired
+    private ChatClient contentGenerator;
+
+    @Autowired
+    private RelevancyEvaluator relevancyEvaluator;
+
+    @Autowired
+    private FactCheckingEvaluator factCheckingEvaluator;
+
+    @Test
+    void whenChatClientProvidesAnswerRelevantToTopic_thenRelevancyEvaluationSucceeds() {
+        String question = "How many days sick leave can I take?";
+        ChatResponse chatResponse = contentGenerator.prompt()
+            .user(question)
+            .call()
+            .chatResponse();
+
+        String answer = chatResponse.getResult().getOutput().getContent();
+        List<Document> documents = chatResponse.getMetadata().get(QuestionAnswerAdvisor.RETRIEVED_DOCUMENTS);
+        EvaluationRequest evaluationRequest = new EvaluationRequest(question, documents, answer);
+
+        EvaluationResponse evaluationResponse = relevancyEvaluator.evaluate(evaluationRequest);
+        assertThat(evaluationResponse.isPass()).isTrue();
+    }
+
+    @Test
+    void whenChatClientProvidesAnswerIrrelevantToTopic_thenRelevancyEvaluationFails() {
+        String question = "How many days sick leave can I take?";
+        ChatResponse chatResponse = contentGenerator.prompt()
+            .user(question)
+            .call()
+            .chatResponse();
+
+        String nonRelevantAnswer = "A lion is the king of the jungle";
+        List<Document> documents = chatResponse.getMetadata().get(QuestionAnswerAdvisor.RETRIEVED_DOCUMENTS);
+        EvaluationRequest evaluationRequest = new EvaluationRequest(question, documents, nonRelevantAnswer);
+
+        EvaluationResponse evaluationResponse = relevancyEvaluator.evaluate(evaluationRequest);
+        assertThat(evaluationResponse.isPass()).isFalse();
+    }
+
+    @Test
+    void whenChatClientProvidesWrongAnswerRelevantToTopic_thenRelevancyEvaluationFails() {
+        String question = "How many days sick leave can I take?";
+        ChatResponse chatResponse = contentGenerator.prompt()
+            .user(question)
+            .call()
+            .chatResponse();
+
+        String wrongAnswer = "You can take no leaves. Get back to work!";
+        List<Document> documents = chatResponse.getMetadata().get(QuestionAnswerAdvisor.RETRIEVED_DOCUMENTS);
+        EvaluationRequest evaluationRequest = new EvaluationRequest(question, documents, wrongAnswer);
+
+        EvaluationResponse evaluationResponse = relevancyEvaluator.evaluate(evaluationRequest);
+        assertThat(evaluationResponse.isPass()).isFalse();
+    }
+
+    @Test
+    void whenChatClientProvidesFactuallyCorrectAnswer_thenFactCheckingEvaluationSucceeds() {
+        String question = "How many days sick leave can I take?";
+        ChatResponse chatResponse = contentGenerator.prompt()
+            .user(question)
+            .call()
+            .chatResponse();
+
+        String answer = chatResponse.getResult().getOutput().getContent();
+        List<Document> documents = chatResponse.getMetadata().get(QuestionAnswerAdvisor.RETRIEVED_DOCUMENTS);
+        EvaluationRequest evaluationRequest = new EvaluationRequest(question, documents, answer);
+
+        EvaluationResponse evaluationResponse = factCheckingEvaluator.evaluate(evaluationRequest);
+        assertThat(evaluationResponse.isPass()).isTrue();
+    }
+
+    @Test
+    void whenChatClientProvidesFactuallyIncorrectAnswer_thenFactCheckingEvaluationFails() {
+        String question = "How many days sick leave can I take?";
+        ChatResponse chatResponse = contentGenerator.prompt()
+            .user(question)
+            .call()
+            .chatResponse();
+
+        String wrongAnswer = "You can take no leaves. Get back to work!";
+        List<Document> documents = chatResponse.getMetadata().get(QuestionAnswerAdvisor.RETRIEVED_DOCUMENTS);
+        EvaluationRequest evaluationRequest = new EvaluationRequest(question, documents, wrongAnswer);
+
+        EvaluationResponse evaluationResponse = factCheckingEvaluator.evaluate(evaluationRequest);
+        assertThat(evaluationResponse.isPass()).isFalse();
+    }
+
+}

spring-ai-2/src/test/java/com/baeldung/springai/evaluator/TestcontainersConfiguration.java

@@ -0,0 +1,23 @@
+package com.baeldung.springai.evaluator;
+
+import org.springframework.boot.test.context.TestConfiguration;
+import org.springframework.context.annotation.Bean;
+import org.springframework.test.context.DynamicPropertyRegistrar;
+import org.testcontainers.ollama.OllamaContainer;
+
+@TestConfiguration(proxyBeanMethods = false)
+class TestcontainersConfiguration {
+
+    @Bean
+    public OllamaContainer ollamaContainer() {
+        return new OllamaContainer("ollama/ollama:0.5.7");
+    }
+
+    @Bean
+    public DynamicPropertyRegistrar dynamicPropertyRegistrar(OllamaContainer ollamaContainer) {
+        return registry -> {
+            registry.add("spring.ai.ollama.base-url", ollamaContainer::getEndpoint);
+        };
+    }
+
+}