diff --git a/spring-ai-core/src/main/java/org/springframework/ai/vectorstore/SimpleVectorStore.java b/spring-ai-core/src/main/java/org/springframework/ai/vectorstore/SimpleVectorStore.java index c9414392086..631784b9649 100644 --- a/spring-ai-core/src/main/java/org/springframework/ai/vectorstore/SimpleVectorStore.java +++ b/spring-ai-core/src/main/java/org/springframework/ai/vectorstore/SimpleVectorStore.java @@ -16,23 +16,6 @@ package org.springframework.ai.vectorstore; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.OutputStream; -import java.io.OutputStreamWriter; -import java.io.Writer; -import java.nio.charset.StandardCharsets; -import java.nio.file.FileAlreadyExistsException; -import java.nio.file.Files; -import java.util.Comparator; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.concurrent.ConcurrentHashMap; - import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; @@ -41,23 +24,35 @@ import io.micrometer.observation.ObservationRegistry; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - import org.springframework.ai.document.Document; import org.springframework.ai.embedding.EmbeddingModel; import org.springframework.ai.observation.conventions.VectorStoreProvider; import org.springframework.ai.observation.conventions.VectorStoreSimilarityMetric; import org.springframework.ai.util.JacksonUtils; +import org.springframework.ai.vectorstore.filter.FilterExpressionConverter; +import org.springframework.ai.vectorstore.filter.converter.SimpleVectorStoreFilterExpressionConverter; import org.springframework.ai.vectorstore.observation.AbstractObservationVectorStore; import org.springframework.ai.vectorstore.observation.VectorStoreObservationContext; import org.springframework.ai.vectorstore.observation.VectorStoreObservationConvention; import org.springframework.core.io.Resource; +import org.springframework.expression.ExpressionParser; +import org.springframework.expression.spel.standard.SpelExpressionParser; +import org.springframework.expression.spel.support.StandardEvaluationContext; + +import java.io.*; +import java.nio.charset.StandardCharsets; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.Files; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Predicate; /** * SimpleVectorStore is a simple implementation of the VectorStore interface. - * + *

* It also provides methods to save the current state of the vectors to a file, and to * load vectors from a file. - * + *

* For a deeper understanding of the mathematical concepts and computations involved in * calculating similarity scores among vectors, refer to this * [resource](https://docs.spring.io/spring-ai/reference/api/vectordbs.html#_understanding_vectors). @@ -68,6 +63,7 @@ * @author Christian Tzolov * @author Sebastien Deleuze * @author Ilayaperumal Gopinathan + * @author Jemin Huh */ public class SimpleVectorStore extends AbstractObservationVectorStore { @@ -75,6 +71,10 @@ public class SimpleVectorStore extends AbstractObservationVectorStore { private final ObjectMapper objectMapper; + private final ExpressionParser expressionParser; + + private final FilterExpressionConverter filterExpressionConverter; + protected Map store = new ConcurrentHashMap<>(); protected EmbeddingModel embeddingModel; @@ -91,6 +91,8 @@ public SimpleVectorStore(EmbeddingModel embeddingModel, ObservationRegistry obse Objects.requireNonNull(embeddingModel, "EmbeddingModel must not be null"); this.embeddingModel = embeddingModel; this.objectMapper = JsonMapper.builder().addModules(JacksonUtils.instantiateAvailableModules()).build(); + this.expressionParser = new SpelExpressionParser(); + this.filterExpressionConverter = new SimpleVectorStoreFilterExpressionConverter(); } @Override @@ -119,14 +121,11 @@ public Optional doDelete(List idList) { @Override public List doSimilaritySearch(SearchRequest request) { - if (request.getFilterExpression() != null) { - throw new UnsupportedOperationException( - "The [" + this.getClass() + "] doesn't support metadata filtering!"); - } - + Predicate documentFilterPredicate = doFilterPredicate(request); float[] userQueryEmbedding = getUserQueryEmbedding(request.getQuery()); return this.store.values() .stream() + .filter(documentFilterPredicate) .map(entry -> new Similarity(entry, EmbeddingMath.cosineSimilarity(userQueryEmbedding, entry.getEmbedding()))) .filter(s -> s.score >= request.getSimilarityThreshold()) @@ -136,6 +135,16 @@ public List doSimilaritySearch(SearchRequest request) { .toList(); } + private Predicate doFilterPredicate(SearchRequest request) { + return request.hasFilterExpression() ? document -> { + StandardEvaluationContext context = new StandardEvaluationContext(); + context.setVariable("metadata", document.getMetadata()); + return this.expressionParser + .parseExpression(this.filterExpressionConverter.convertExpression(request.getFilterExpression())) + .getValue(context, Boolean.class); + } : document -> true; + } + /** * Serialize the vector store content into a file in JSON format. * @param file the file to save the vector store content @@ -247,10 +256,13 @@ public Similarity(SimpleVectorStoreContent content, double score) { } Document getDocument() { + // Add the calculated distance (1 - score) to the metadata + Map metadata = new HashMap<>(this.content.getMetadata()); + metadata.put("distance", 1 - score); return Document.builder() .withId(this.content.getId()) .withContent(this.content.getContent()) - .withMetadata(this.content.getMetadata()) + .withMetadata(metadata) .build(); } diff --git a/spring-ai-core/src/main/java/org/springframework/ai/vectorstore/filter/converter/SimpleVectorStoreFilterExpressionConverter.java b/spring-ai-core/src/main/java/org/springframework/ai/vectorstore/filter/converter/SimpleVectorStoreFilterExpressionConverter.java new file mode 100644 index 00000000000..534507aac26 --- /dev/null +++ b/spring-ai-core/src/main/java/org/springframework/ai/vectorstore/filter/converter/SimpleVectorStoreFilterExpressionConverter.java @@ -0,0 +1,147 @@ +/* + * Copyright 2023-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.vectorstore.filter.converter; + +import org.springframework.ai.vectorstore.filter.Filter; +import org.springframework.ai.vectorstore.filter.Filter.Expression; + +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.List; +import java.util.TimeZone; +import java.util.regex.Pattern; + +/** + * Converts {@link Expression} into SpEL metadata filter expression format. + * (https://docs.spring.io/spring-framework/reference/core/expressions.html) + * + * @author Jemin Huh + */ +public class SimpleVectorStoreFilterExpressionConverter extends AbstractFilterExpressionConverter { + + private static final Pattern DATE_FORMAT_PATTERN = Pattern.compile("\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}Z"); + + private final SimpleDateFormat dateFormat; + + public SimpleVectorStoreFilterExpressionConverter() { + this.dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); + this.dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + } + + @Override + protected void doExpression(Filter.Expression expression, StringBuilder context) { + this.convertOperand(expression.left(), context); + context.append(getOperationSymbol(expression)); + this.convertOperand(expression.right(), context); + } + + private String getOperationSymbol(Filter.Expression exp) { + return switch (exp.type()) { + case AND -> " and "; + case OR -> " or "; + case EQ -> " == "; + case LT -> " < "; + case LTE -> " <= "; + case GT -> " > "; + case GTE -> " >= "; + case NE -> " != "; + case IN -> " in "; + case NIN -> " not in "; + default -> throw new RuntimeException("Not supported expression type: " + exp.type()); + }; + } + + @Override + protected void doKey(Filter.Key key, StringBuilder context) { + var identifier = hasOuterQuotes(key.key()) ? removeOuterQuotes(key.key()) : key.key(); + context.append("#metadata['").append(identifier).append("']"); + } + + @Override + protected void doValue(Filter.Value filterValue, StringBuilder context) { + if (filterValue.value() instanceof List list) { + var formattedList = new StringBuilder("{"); + int c = 0; + for (Object v : list) { + this.doSingleValue(v, formattedList); + if (c++ < list.size() - 1) { + this.doAddValueRangeSpitter(filterValue, formattedList); + } + } + formattedList.append("}"); + + if (context.lastIndexOf("in ") == -1) { + context.append(formattedList); + } + else { + appendSpELContains(formattedList, context); + } + } + else { + this.doSingleValue(filterValue.value(), context); + } + } + + private void appendSpELContains(StringBuilder formattedList, StringBuilder context) { + int metadataStart = context.lastIndexOf("#metadata"); + if (metadataStart == -1) + throw new RuntimeException("Wrong SpEL expression: " + context); + + int metadataEnd = context.indexOf(" ", metadataStart); + String metadata = context.substring(metadataStart, metadataEnd); + context.setLength(context.lastIndexOf("in ")); + context.delete(metadataStart, metadataEnd + 1); + context.append(formattedList).append(".contains(").append(metadata).append(")"); + } + + @Override + protected void doSingleValue(Object value, StringBuilder context) { + if (value instanceof Date date) { + context.append("'"); + context.append(this.dateFormat.format(date)); + context.append("'"); + } + else if (value instanceof String text) { + context.append("'"); + if (DATE_FORMAT_PATTERN.matcher(text).matches()) { + try { + Date date = this.dateFormat.parse(text); + context.append(this.dateFormat.format(date)); + } + catch (ParseException e) { + throw new IllegalArgumentException("Invalid date type:" + text, e); + } + } + else { + context.append(text); + } + context.append("'"); + } + else { + context.append(value); + } + } + + @Override + protected void doGroup(Filter.Group group, StringBuilder context) { + context.append("("); + super.doGroup(group, context); + context.append(")"); + } + +} diff --git a/spring-ai-core/src/test/java/org/springframework/ai/vectorstore/SimpleVectorStoreWithFilterTests.java b/spring-ai-core/src/test/java/org/springframework/ai/vectorstore/SimpleVectorStoreWithFilterTests.java new file mode 100644 index 00000000000..a1a5f6e64f2 --- /dev/null +++ b/spring-ai-core/src/test/java/org/springframework/ai/vectorstore/SimpleVectorStoreWithFilterTests.java @@ -0,0 +1,220 @@ +/* + * Copyright 2023-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.vectorstore; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.CleanupMode; +import org.junit.jupiter.api.io.TempDir; +import org.springframework.ai.document.Document; +import org.springframework.ai.embedding.EmbeddingModel; +import org.springframework.ai.vectorstore.filter.Filter; + +import java.nio.file.Path; +import java.util.Arrays; +import java.util.Date; +import java.util.List; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.springframework.ai.vectorstore.filter.Filter.ExpressionType.*; + +/** + * @author Jemin Huh + */ +class SimpleVectorStoreWithFilterTests { + + @TempDir(cleanup = CleanupMode.ON_SUCCESS) + Path tempDir; + + private SimpleVectorStore vectorStore; + + private EmbeddingModel mockEmbeddingModel; + + @BeforeEach + void setUp() { + this.mockEmbeddingModel = mock(EmbeddingModel.class); + when(this.mockEmbeddingModel.dimensions()).thenReturn(3); + when(this.mockEmbeddingModel.embed(any(String.class))).thenReturn(new float[] { 0.1f, 0.2f, 0.3f }); + when(this.mockEmbeddingModel.embed(any(Document.class))).thenReturn(new float[] { 0.1f, 0.2f, 0.3f }); + this.vectorStore = new SimpleVectorStore(this.mockEmbeddingModel); + } + + @Test + void shouldAddAndRetrieveDocumentWithFilter() { + Document doc = Document.builder() + .withId("1") + .withContent("test content") + .withMetadata(Map.of("country", "BG", "year", 2020, "activationDate", "1970-01-01T00:00:02Z")) + .build(); + + this.vectorStore.add(List.of(doc)); + + List results = this.vectorStore + .similaritySearch(SearchRequest.query("test content").withFilterExpression("country == 'BG'")); + assertThat(results).hasSize(1).first().satisfies(result -> { + assertThat(result.getId()).isEqualTo("1"); + assertThat(result.getContent()).isEqualTo("test content"); + assertThat(result.getMetadata()).hasSize(4); + assertThat(result.getMetadata()).containsEntry("distance", 2.220446049250313E-16); + }); + + results = this.vectorStore + .similaritySearch(SearchRequest.query("test content").withFilterExpression("country == 'KR'")); + assertThat(results).hasSize(0); + + results = this.vectorStore.similaritySearch( + SearchRequest.query("test content").withFilterExpression("country == 'BG' && year == 2020")); + assertThat(results).hasSize(1).first().satisfies(result -> { + assertThat(result.getId()).isEqualTo("1"); + assertThat(result.getContent()).isEqualTo("test content"); + assertThat(result.getMetadata()).hasSize(4); + assertThat(result.getMetadata()).containsEntry("distance", 2.220446049250313E-16); + }); + + results = this.vectorStore.similaritySearch( + SearchRequest.query("test content").withFilterExpression("country == 'BG' && year == 2024")); + assertThat(results).hasSize(0); + + results = this.vectorStore + .similaritySearch(SearchRequest.query("test content").withFilterExpression("country in ['BG', 'NL']")); + assertThat(results).hasSize(1).first().satisfies(result -> { + assertThat(result.getId()).isEqualTo("1"); + assertThat(result.getContent()).isEqualTo("test content"); + assertThat(result.getMetadata()).hasSize(4); + assertThat(result.getMetadata()).containsEntry("distance", 2.220446049250313E-16); + }); + + results = this.vectorStore + .similaritySearch(SearchRequest.query("test content").withFilterExpression("country in ['KR', 'NL']")); + assertThat(results).hasSize(0); + + results = this.vectorStore.similaritySearch(SearchRequest.query("test content") + .withFilterExpression( + new Filter.Expression(EQ, new Filter.Key("activationDate"), new Filter.Value(new Date(2000))))); + assertThat(results).hasSize(1).first().satisfies(result -> { + assertThat(result.getId()).isEqualTo("1"); + assertThat(result.getContent()).isEqualTo("test content"); + assertThat(result.getMetadata()).hasSize(4); + assertThat(result.getMetadata()).containsEntry("distance", 2.220446049250313E-16); + }); + + results = this.vectorStore.similaritySearch(SearchRequest.query("test content") + .withFilterExpression( + new Filter.Expression(EQ, new Filter.Key("activationDate"), new Filter.Value(new Date(3000))))); + assertThat(results).hasSize(0); + + } + + @Test + void shouldAddMultipleDocumentsWithFilter() { + List docs = Arrays.asList( + Document.builder() + .withId("1") + .withContent("first") + .withMetadata(Map.of("country", "BG", "year", 2020, "activationDate", "1970-01-01T00:00:02Z")) + .build(), + Document.builder() + .withId("2") + .withContent("second") + .withMetadata(Map.of("country", "KR", "year", 2022, "activationDate", "1970-01-01T00:00:03Z")) + .build()); + + this.vectorStore.add(docs); + + List results = this.vectorStore.similaritySearch("first"); + assertThat(results).hasSize(2).extracting(Document::getId).containsExactlyInAnyOrder("1", "2"); + + results = this.vectorStore + .similaritySearch(SearchRequest.query("first").withFilterExpression("country == 'BG'")); + assertThat(results).hasSize(1).first().satisfies(result -> { + assertThat(result.getId()).isEqualTo("1"); + assertThat(result.getContent()).isEqualTo("first"); + assertThat(result.getMetadata()).hasSize(4); + assertThat(result.getMetadata()).containsEntry("distance", 2.220446049250313E-16); + }); + + results = this.vectorStore + .similaritySearch(SearchRequest.query("first").withFilterExpression("country == 'NL'")); + assertThat(results).hasSize(0); + + results = this.vectorStore + .similaritySearch(SearchRequest.query("first").withFilterExpression("country == 'BG' && year == 2020")); + assertThat(results).hasSize(1).first().satisfies(result -> { + assertThat(result.getId()).isEqualTo("1"); + assertThat(result.getContent()).isEqualTo("first"); + assertThat(result.getMetadata()).hasSize(4); + assertThat(result.getMetadata()).containsEntry("distance", 2.220446049250313E-16); + }); + + results = this.vectorStore + .similaritySearch(SearchRequest.query("first").withFilterExpression("country == 'KR' && year == 2022")); + assertThat(results).hasSize(1).first().satisfies(result -> { + assertThat(result.getId()).isEqualTo("2"); + assertThat(result.getContent()).isEqualTo("second"); + assertThat(result.getMetadata()).hasSize(4); + assertThat(result.getMetadata()).containsEntry("distance", 2.220446049250313E-16); + }); + + results = this.vectorStore.similaritySearch( + SearchRequest.query("test content").withFilterExpression("country == 'KR' && year == 2024")); + assertThat(results).hasSize(0); + + results = this.vectorStore + .similaritySearch(SearchRequest.query("first").withFilterExpression("country in ['BG', 'NL']")); + assertThat(results).hasSize(1).first().satisfies(result -> { + assertThat(result.getId()).isEqualTo("1"); + assertThat(result.getContent()).isEqualTo("first"); + assertThat(result.getMetadata()).hasSize(4); + assertThat(result.getMetadata()).containsEntry("distance", 2.220446049250313E-16); + }); + + results = this.vectorStore + .similaritySearch(SearchRequest.query("first").withFilterExpression("country in ['KR', 'NL']")); + assertThat(results).hasSize(1); + + results = this.vectorStore.similaritySearch(SearchRequest.query("first") + .withFilterExpression( + new Filter.Expression(EQ, new Filter.Key("activationDate"), new Filter.Value(new Date(2000))))); + assertThat(results).hasSize(1).first().satisfies(result -> { + assertThat(result.getId()).isEqualTo("1"); + assertThat(result.getContent()).isEqualTo("first"); + assertThat(result.getMetadata()).hasSize(4); + assertThat(result.getMetadata()).containsEntry("distance", 2.220446049250313E-16); + }); + + results = this.vectorStore.similaritySearch(SearchRequest.query("first") + .withFilterExpression(new Filter.Expression(AND, + new Filter.Expression(GTE, new Filter.Key("activationDate"), new Filter.Value(new Date(2000))), + new Filter.Expression(LTE, new Filter.Key("activationDate"), new Filter.Value(new Date(3000)))))); + assertThat(results).hasSize(2).first().satisfies(result -> { + assertThat(result.getId()).isEqualTo("1"); + assertThat(result.getContent()).isEqualTo("first"); + assertThat(result.getMetadata()).hasSize(4); + assertThat(result.getMetadata()).containsEntry("distance", 2.220446049250313E-16); + }); + + results = this.vectorStore.similaritySearch(SearchRequest.query("test content") + .withFilterExpression( + new Filter.Expression(EQ, new Filter.Key("activationDate"), new Filter.Value(new Date(3000))))); + assertThat(results).hasSize(1); + } + +} diff --git a/spring-ai-core/src/test/java/org/springframework/ai/vectorstore/filter/converter/SimpleVectorStoreFilterExpressionConverterTests.java b/spring-ai-core/src/test/java/org/springframework/ai/vectorstore/filter/converter/SimpleVectorStoreFilterExpressionConverterTests.java new file mode 100644 index 00000000000..7179d7fe552 --- /dev/null +++ b/spring-ai-core/src/test/java/org/springframework/ai/vectorstore/filter/converter/SimpleVectorStoreFilterExpressionConverterTests.java @@ -0,0 +1,199 @@ +/* + * Copyright 2023-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.vectorstore.filter.converter; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.springframework.ai.vectorstore.filter.Filter; +import org.springframework.ai.vectorstore.filter.FilterExpressionConverter; +import org.springframework.expression.ExpressionParser; +import org.springframework.expression.spel.standard.SpelExpressionParser; +import org.springframework.expression.spel.support.StandardEvaluationContext; + +import java.util.Date; +import java.util.List; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.springframework.ai.vectorstore.filter.Filter.ExpressionType.*; + +/** + * @author Jemin Huh + */ +public class SimpleVectorStoreFilterExpressionConverterTests { + + final FilterExpressionConverter converter = new SimpleVectorStoreFilterExpressionConverter(); + + @Test + public void testDate() { + String vectorExpr = this.converter.convertExpression(new Filter.Expression(EQ, new Filter.Key("activationDate"), + new Filter.Value(new Date(1704637752148L)))); + assertThat(vectorExpr).isEqualTo("#metadata['activationDate'] == '2024-01-07T14:29:12Z'"); + + StandardEvaluationContext context = new StandardEvaluationContext(); + ExpressionParser parser = new SpelExpressionParser(); + context.setVariable("metadata", + Map.of("activationDate", "2024-01-07T14:29:12Z", "year", 2020, "country", "BG")); + Assertions.assertTrue(parser.parseExpression(vectorExpr).getValue(context, Boolean.class)); + + vectorExpr = this.converter.convertExpression( + new Filter.Expression(EQ, new Filter.Key("activationDate"), new Filter.Value("1970-01-01T00:00:02Z"))); + assertThat(vectorExpr).isEqualTo("#metadata['activationDate'] == '1970-01-01T00:00:02Z'"); + + context.setVariable("metadata", + Map.of("activationDate", "1970-01-01T00:00:02Z", "year", 2020, "country", "BG")); + Assertions.assertTrue(parser.parseExpression(vectorExpr).getValue(context, Boolean.class)); + + } + + @Test + public void testEQ() { + String vectorExpr = this.converter + .convertExpression(new Filter.Expression(EQ, new Filter.Key("country"), new Filter.Value("BG"))); + assertThat(vectorExpr).isEqualTo("#metadata['country'] == 'BG'"); + + StandardEvaluationContext context = new StandardEvaluationContext(); + ExpressionParser parser = new SpelExpressionParser(); + context.setVariable("metadata", Map.of("city", "Seoul", "year", 2020, "country", "BG")); + Assertions.assertTrue(parser.parseExpression(vectorExpr).getValue(context, Boolean.class)); + + } + + @Test + public void tesEqAndGte() { + String vectorExpr = this.converter.convertExpression(new Filter.Expression(AND, + new Filter.Expression(EQ, new Filter.Key("genre"), new Filter.Value("drama")), + new Filter.Expression(GTE, new Filter.Key("year"), new Filter.Value(2020)))); + assertThat(vectorExpr).isEqualTo("#metadata['genre'] == 'drama' and #metadata['year'] >= 2020"); + + StandardEvaluationContext context = new StandardEvaluationContext(); + ExpressionParser parser = new SpelExpressionParser(); + context.setVariable("metadata", Map.of("genre", "drama", "year", 2020, "country", "BG")); + Assertions.assertTrue(parser.parseExpression(vectorExpr).getValue(context, Boolean.class)); + + } + + @Test + public void tesIn() { + String vectorExpr = this.converter.convertExpression(new Filter.Expression(IN, new Filter.Key("genre"), + new Filter.Value(List.of("comedy", "documentary", "drama")))); + assertThat(vectorExpr).isEqualTo("{'comedy','documentary','drama'}.contains(#metadata['genre'])"); + + StandardEvaluationContext context = new StandardEvaluationContext(); + ExpressionParser parser = new SpelExpressionParser(); + context.setVariable("metadata", Map.of("genre", "drama", "year", 2020, "country", "BG")); + Assertions.assertTrue(parser.parseExpression(vectorExpr).getValue(context, Boolean.class)); + + } + + @Test + public void testNe() { + String vectorExpr = this.converter.convertExpression( + new Filter.Expression(OR, new Filter.Expression(GTE, new Filter.Key("year"), new Filter.Value(2020)), + new Filter.Expression(AND, + new Filter.Expression(EQ, new Filter.Key("country"), new Filter.Value("BG")), + new Filter.Expression(NE, new Filter.Key("city"), new Filter.Value("Sofia"))))); + assertThat(vectorExpr) + .isEqualTo("#metadata['year'] >= 2020 or #metadata['country'] == 'BG' and #metadata['city'] != 'Sofia'"); + + StandardEvaluationContext context = new StandardEvaluationContext(); + ExpressionParser parser = new SpelExpressionParser(); + context.setVariable("metadata", Map.of("city", "Seoul", "year", 2020, "country", "BG")); + Assertions.assertTrue(parser.parseExpression(vectorExpr).getValue(context, Boolean.class)); + + } + + @Test + public void testGroup() { + String vectorExpr = this.converter.convertExpression(new Filter.Expression(AND, + new Filter.Group(new Filter.Expression(OR, + new Filter.Expression(GTE, new Filter.Key("year"), new Filter.Value(2020)), + new Filter.Expression(EQ, new Filter.Key("country"), new Filter.Value("BG")))), + new Filter.Expression(NIN, new Filter.Key("city"), new Filter.Value(List.of("Sofia", "Plovdiv"))))); + assertThat(vectorExpr).isEqualTo( + "(#metadata['year'] >= 2020 or #metadata['country'] == 'BG') and not {'Sofia','Plovdiv'}.contains(#metadata['city'])"); + + StandardEvaluationContext context = new StandardEvaluationContext(); + ExpressionParser parser = new SpelExpressionParser(); + context.setVariable("metadata", Map.of("city", "Seoul", "year", 2020, "country", "BG")); + Assertions.assertTrue(parser.parseExpression(vectorExpr).getValue(context, Boolean.class)); + + } + + @Test + public void tesBoolean() { + String vectorExpr = this.converter.convertExpression(new Filter.Expression(AND, + new Filter.Expression(AND, new Filter.Expression(EQ, new Filter.Key("isOpen"), new Filter.Value(true)), + new Filter.Expression(GTE, new Filter.Key("year"), new Filter.Value(2020))), + new Filter.Expression(IN, new Filter.Key("country"), new Filter.Value(List.of("BG", "NL", "US"))))); + + assertThat(vectorExpr).isEqualTo( + "#metadata['isOpen'] == true and #metadata['year'] >= 2020 and {'BG','NL','US'}.contains(#metadata['country'])"); + + StandardEvaluationContext context = new StandardEvaluationContext(); + ExpressionParser parser = new SpelExpressionParser(); + context.setVariable("metadata", Map.of("isOpen", true, "year", 2020, "country", "NL")); + Assertions.assertTrue(parser.parseExpression(vectorExpr).getValue(context, Boolean.class)); + + vectorExpr = this.converter.convertExpression(new Filter.Expression(AND, + new Filter.Expression(AND, new Filter.Expression(EQ, new Filter.Key("isOpen"), new Filter.Value(true)), + new Filter.Expression(GTE, new Filter.Key("year"), new Filter.Value(2020))), + new Filter.Expression(NIN, new Filter.Key("country"), new Filter.Value(List.of("BG", "NL", "US"))))); + + assertThat(vectorExpr).isEqualTo( + "#metadata['isOpen'] == true and #metadata['year'] >= 2020 and not {'BG','NL','US'}.contains(#metadata['country'])"); + + context.setVariable("metadata", Map.of("isOpen", true, "year", 2020, "country", "KR")); + Assertions.assertTrue(parser.parseExpression(vectorExpr).getValue(context, Boolean.class)); + } + + @Test + public void testDecimal() { + String vectorExpr = this.converter.convertExpression(new Filter.Expression(AND, + new Filter.Expression(GTE, new Filter.Key("temperature"), new Filter.Value(-15.6)), + new Filter.Expression(LTE, new Filter.Key("temperature"), new Filter.Value(20.13)))); + + assertThat(vectorExpr).isEqualTo("#metadata['temperature'] >= -15.6 and #metadata['temperature'] <= 20.13"); + + StandardEvaluationContext context = new StandardEvaluationContext(); + ExpressionParser parser = new SpelExpressionParser(); + context.setVariable("metadata", Map.of("temperature", -15.6)); + Assertions.assertTrue(parser.parseExpression(vectorExpr).getValue(context, Boolean.class)); + context.setVariable("metadata", Map.of("temperature", 20.13)); + Assertions.assertTrue(parser.parseExpression(vectorExpr).getValue(context, Boolean.class)); + context.setVariable("metadata", Map.of("temperature", -1.6)); + Assertions.assertTrue(parser.parseExpression(vectorExpr).getValue(context, Boolean.class)); + + } + + @Test + public void testComplexIdentifiers() { + String vectorExpr = this.converter + .convertExpression(new Filter.Expression(EQ, new Filter.Key("\"country 1 2 3\""), new Filter.Value("BG"))); + assertThat(vectorExpr).isEqualTo("#metadata['country 1 2 3'] == 'BG'"); + + vectorExpr = this.converter + .convertExpression(new Filter.Expression(EQ, new Filter.Key("'country 1 2 3'"), new Filter.Value("BG"))); + assertThat(vectorExpr).isEqualTo("#metadata['country 1 2 3'] == 'BG'"); + + StandardEvaluationContext context = new StandardEvaluationContext(); + ExpressionParser parser = new SpelExpressionParser(); + context.setVariable("metadata", Map.of("country 1 2 3", "BG")); + Assertions.assertTrue(parser.parseExpression(vectorExpr).getValue(context, Boolean.class)); + } + +}